CodeMirror/czkawka
1
0
Fork 0
mirror of synced 2024-04-29 01:52:39 +12:00
Code Issues Projects Releases Wiki Activity

Random updates and modifications (#1070)

Browse source 
* Updates

* Rust embed

* GTK 4

* Handsome logger

* Handsome logger

* Lock

* DB

* Logging

* It compile

* Cleanup

* Sl

* Sl

* Slint

* 1.70.0

* Appimage
This commit is contained in:
Rafał Mikrut 2023-10-05 08:06:47 +02:00 committed by GitHub
parent 30068b58d3
commit edfc8e7b5f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
46 changed files with 2187 additions and 2236 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/linux_cli.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
linux-cli:
strategy:
matrix:
toolchain: [ stable, 1.67.1 ]
toolchain: [ stable, 1.70.0 ]
type: [ release ]
runs-on: ubuntu-20.04
steps:

10
.github/workflows/linux_gui.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
linux-gui:
strategy:
matrix:
toolchain: [ stable, 1.67.1 ]
toolchain: [ stable, 1.70.0 ]
type: [ release ]
runs-on: ubuntu-22.04
steps:
@ -29,7 +29,7 @@ jobs:
env:
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C debuginfo=0"
if: ${{ (matrix.type == 'release') && (matrix.toolchain == '1.67.1') }}
if: ${{ (matrix.type == 'release') && (matrix.toolchain == '1.70.0') }}
- name: Store Linux GUI Heif
uses: actions/upload-artifact@v3
@ -97,9 +97,9 @@ jobs:
rm -rf czkawka_gui
cp target/release/czkawka_gui .
strip czkawka_gui
wget https://github.com/AppImage/pkg2appimage/releases/download/continuous/pkg2appimage-1807-x86_64.AppImage
chmod +x pkg2appimage-1807-x86_64.AppImage
./pkg2appimage-1807-x86_64.AppImage misc/czkawka-appimage-recipe.yml
wget https://github.com/AppImageCommunity/pkg2appimage/releases/download/continuous/pkg2appimage--x86_64.AppImage -O pkg2.appimage
chmod +x ./pkg2.appimage
./pkg2.appimage misc/czkawka-appimage-recipe.yml
mv out/Czkawka*.AppImage out/czkawka_gui-minimal.AppImage
- name: Minimal Appimage Upload

819
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

6
Cargo.toml
View file

@ -3,7 +3,13 @@ members = [
"czkawka_core",
"czkawka_cli",
"czkawka_gui",
# "czkawka_slint_gui",
]
exclude = [
"czkawka_slint_gui"
]
resolver = "2"
[profile.release]
# panic = "unwind" in opposite to "abort", allows to catch panic!()
# Since Czkawka parse different types of files with few libraries, it is possible

3
Changelog.md
View file

@ -1,3 +1,6 @@
## Version 6.1.0 - ?
## Version 6.0.0 - 11.06.2023r
- Add finding similar audio files by content - [#970](https://github.com/qarmin/czkawka/pull/970)
- Allow to find duplicates by name/size at once - [#956](https://github.com/qarmin/czkawka/pull/956)

9
czkawka_cli/Cargo.toml
View file

@ -3,7 +3,7 @@ name = "czkawka_cli"
version = "6.0.0"
authors = ["Rafał Mikrut <mikrutrafal@protonmail.com>"]
edition = "2021"
rust-version = "1.67.1"
rust-version = "1.70.0"
description = "CLI frontend of Czkawka"
license = "MIT"
homepage = "https://github.com/qarmin/czkawka"
@ -15,10 +15,9 @@ clap = { version = "4.3", features = ["derive"] }
# For enum types
image_hasher = "1.2"
[dependencies.czkawka_core]
path = "../czkawka_core"
version = "6.0.0"
features = []
log = "0.4.20"
handsome_logger = "0.8"
czkawka_core = { path = "../czkawka_core", version = "6.0.0", features = [] }
[features]
default = []

5
czkawka_cli/src/main.rs
View file

@ -8,7 +8,8 @@ use commands::Commands;
use czkawka_core::bad_extensions::BadExtensions;
use czkawka_core::big_file::{self, BigFile, SearchMode};
use czkawka_core::broken_files::{self, BrokenFiles};
use czkawka_core::common::set_number_of_threads;
use czkawka_core::common::{set_number_of_threads, setup_logger};
use czkawka_core::common_tool::CommonData;
#[allow(unused_imports)] // It is used in release for print_results().
use czkawka_core::common_traits::*;
use czkawka_core::duplicate::DuplicateFinder;
@ -30,6 +31,8 @@ mod commands;
fn main() {
let command = Args::parse().command;
setup_logger(true);
#[cfg(debug_assertions)]
println!("{command:?}");

21
czkawka_core/Cargo.toml
View file

@ -3,7 +3,7 @@ name = "czkawka_core"
version = "6.0.0"
authors = ["Rafał Mikrut <mikrutrafal@protonmail.com>"]
edition = "2021"
rust-version = "1.67.1"
rust-version = "1.70.0"
description = "Core of Czkawka app"
license = "MIT"
homepage = "https://github.com/qarmin/czkawka"
@ -12,7 +12,7 @@ repository = "https://github.com/qarmin/czkawka"
[dependencies]
humansize = "2.1"
rayon = "1.7"
rayon = "1.8"
crossbeam-channel = "0.5"
# For saving/loading config files to specific directories
@ -25,8 +25,8 @@ image = "0.24"
hamming = "0.1"
# Needed by same music
bitflags = "2.3"
lofty = "0.14"
bitflags = "2.4"
lofty = "0.16"
# Futures - needed by async progress sender
futures = "0.3.28"
@ -41,11 +41,11 @@ rusty-chromaprint = "0.1"
symphonia = { version = "0.5", features = ["all"] }
# Hashes for duplicate files
blake3 = "1.4"
blake3 = "1.5"
crc32fast = "1.3"
xxhash-rust = { version = "0.8", features = ["xxh3"] }
tempfile = "3.6"
tempfile = "3.8"
# Video Duplicates
vid_dup_finder_lib = "0.1"
@ -57,9 +57,9 @@ bincode = "1.3"
serde_json = "1.0"
# Language
i18n-embed = { version = "0.13", features = ["fluent-system", "desktop-requester"] }
i18n-embed-fl = "0.6"
rust-embed = "6.8"
i18n-embed = { version = "0.14", features = ["fluent-system", "desktop-requester"] }
i18n-embed-fl = "0.7"
rust-embed = "8.0"
once_cell = "1.18"
# Raw image files
@ -79,6 +79,9 @@ anyhow = { version = "1.0" }
state = "0.6"
log = "0.4.20"
handsome_logger = "0.8"
[features]
default = []
heif = ["dep:libheif-rs", "dep:libheif-sys"]

202
czkawka_core/src/bad_extensions.rs
View file

@ -9,15 +9,13 @@ use std::sync::Arc;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use log::{debug, info};
use mime_guess::get_mime_extensions;
use rayon::prelude::*;
use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::*;
static DISABLED_EXTENSIONS: &[&str] = &["file", "cache", "bak", "data"]; // Such files can have any type inside
@ -72,7 +70,15 @@ const WORKAROUNDS: &[(&str, &str)] = &[
("xml", "mum"),
("xml", "resx"),
("zip", "wmz"),
// Games specific extensions - cannot be used here common extensions like zip
("gz", "h3m"), // Heroes 3
("zip", "hashdb"), // Gog
("c2", "zip"), // King of the Dark Age
("c2", "bmp"), // King of the Dark Age
("c2", "avi"), // King of the Dark Age
("c2", "exe"), // King of the Dark Age
// Other
("der", "keystore"), // Godot/Android keystore
("exe", "pyd"), // Python/Mingw
("gz", "blend"), // Blender
("gz", "crate"), // Cargo
@ -81,6 +87,7 @@ const WORKAROUNDS: &[(&str, &str)] = &[
("html", "dtd"), // Mingw
("html", "ent"), // Mingw
("html", "md"), // Markdown
("html", "svelte"), // Svelte
("jpg", "jfif"), // Photo format
("m4v", "mp4"), // m4v and mp4 are interchangeable
("mobi", "azw3"), // Ebook format
@ -91,6 +98,7 @@ const WORKAROUNDS: &[(&str, &str)] = &[
("ods", "ots"), // Libreoffice
("odt", "ott"), // Libreoffice
("ogg", "ogv"), // Audio format
("pem", "key"), // curl, openssl
("pptx", "ppsx"), // Powerpoint
("sh", "bash"), // Linux
("sh", "guess"), // GNU
@ -109,6 +117,7 @@ const WORKAROUNDS: &[(&str, &str)] = &[
("xml", "dae"), // 3D models
("xml", "docbook"), //
("xml", "fb2"), //
("xml", "filters"), // Visual studio
("xml", "gir"), // GTK
("xml", "glade"), // Glade
("xml", "iml"), // Intelij Idea
@ -165,146 +174,67 @@ pub struct Info {
pub number_of_files_with_bad_extension: usize,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
pub struct BadExtensions {
tool_type: ToolType,
text_messages: Messages,
common_data: CommonToolData,
information: Info,
files_to_check: Vec<FileEntry>,
bad_extensions_files: Vec<BadFileEntry>,
directories: Directories,
allowed_extensions: Extensions,
excluded_items: ExcludedItems,
minimal_file_size: u64,
maximal_file_size: u64,
recursive_search: bool,
stopped_search: bool,
save_also_as_json: bool,
include_files_without_extension: bool,
}
impl BadExtensions {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::BadExtensions,
text_messages: Messages::new(),
information: Info::new(),
recursive_search: true,
allowed_extensions: Extensions::new(),
directories: Directories::new(),
excluded_items: ExcludedItems::new(),
common_data: CommonToolData::new(ToolType::BadExtensions),
information: Info::default(),
files_to_check: Default::default(),
stopped_search: false,
minimal_file_size: 8192,
maximal_file_size: u64::MAX,
bad_extensions_files: Default::default(),
save_also_as_json: false,
include_files_without_extension: true,
}
}
pub fn find_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.directories.optimize_directories(self.recursive_search, &mut self.text_messages);
info!("Starting finding files with bad extensions");
let start_time = std::time::Instant::now();
self.find_bad_extensions_files_internal(stop_receiver, progress_sender);
info!("Ended finding files with bad extensions which took {:?}", start_time.elapsed());
}
fn find_bad_extensions_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
if !self.look_for_bad_extensions_files(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
self.debug_print();
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
#[must_use]
pub const fn get_bad_extensions_files(&self) -> &Vec<BadFileEntry> {
&self.bad_extensions_files
}
pub fn set_maximal_file_size(&mut self, maximal_file_size: u64) {
self.maximal_file_size = match maximal_file_size {
0 => 1,
t => t,
};
}
pub fn set_minimal_file_size(&mut self, minimal_file_size: u64) {
self.minimal_file_size = match minimal_file_size {
0 => 1,
t => t,
};
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_save_also_as_json(&mut self, save_also_as_json: bool) {
self.save_also_as_json = save_also_as_json;
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) -> bool {
self.directories.set_included_directory(included_directory, &mut self.text_messages)
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
pub fn set_allowed_extensions(&mut self, allowed_extensions: String) {
self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages);
}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let result = DirTraversalBuilder::new()
.root_dirs(self.directories.included_directories.clone())
.root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ())
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.minimal_file_size(self.minimal_file_size)
.maximal_file_size(self.maximal_file_size)
.directories(self.directories.clone())
.allowed_extensions(self.allowed_extensions.clone())
.excluded_items(self.excluded_items.clone())
.recursive_search(self.recursive_search)
.minimal_file_size(self.common_data.minimal_file_size)
.maximal_file_size(self.common_data.maximal_file_size)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.build()
.run();
match result {
debug!("check_files - collected files");
let res = match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(files_to_check) = grouped_file_entries.get(&()) {
self.files_to_check = files_to_check.clone();
}
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
true
}
@ -312,12 +242,15 @@ impl BadExtensions {
unreachable!()
}
DirTraversalResult::Stopped => false,
}
};
debug!("check_files - end");
res
}
fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("look_for_bad_extensions_files - start");
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, self.files_to_check.len(), CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 1, 1, self.files_to_check.len(), CheckingMethod::None, self.get_cd().tool_type);
let files_to_check = mem::take(&mut self.files_to_check);
@ -327,7 +260,7 @@ impl BadExtensions {
// if hashmap_workarounds.contains_key(found) {
// panic!("Already have {} key", found);
// }
hashmap_workarounds.entry(found).or_insert_with(Vec::new).push(proper);
hashmap_workarounds.entry(found).or_default().push(proper);
}
self.bad_extensions_files = self.verify_extensions(files_to_check, &atomic_counter, stop_receiver, &check_was_stopped, &hashmap_workarounds);
@ -344,6 +277,7 @@ impl BadExtensions {
// Clean unused data
self.files_to_check = Default::default();
debug!("look_for_bad_extensions_files - end");
true
}
@ -355,7 +289,8 @@ impl BadExtensions {
check_was_stopped: &AtomicBool,
hashmap_workarounds: &HashMap<&str, Vec<&str>>,
) -> Vec<BadFileEntry> {
files_to_check
debug!("verify_extensions - start");
let res = files_to_check
.into_par_iter()
.map(|file_entry| {
atomic_counter.fetch_add(1, Ordering::Relaxed);
@ -404,7 +339,9 @@ impl BadExtensions {
.while_some()
.filter(Option::is_some)
.map(Option::unwrap)
.collect::<Vec<_>>()
.collect::<Vec<_>>();
debug!("verify_extensions - end");
res
}
fn get_and_validate_extension(&self, file_entry: &FileEntry, proper_extension: &str) -> Option<String> {
@ -489,18 +426,7 @@ impl DebugPrint for BadExtensions {
return;
}
println!("---------------DEBUG PRINT---------------");
println!("### Information's");
println!("Errors size - {}", self.text_messages.errors.len());
println!("Warnings size - {}", self.text_messages.warnings.len());
println!("Messages size - {}", self.text_messages.messages.len());
println!("### Other");
println!("Excluded items - {:?}", self.excluded_items.items);
println!("Included directories - {:?}", self.directories.included_directories);
println!("Excluded directories - {:?}", self.directories.excluded_directories);
println!("Recursive search - {}", self.recursive_search);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -515,7 +441,7 @@ impl SaveResults for BadExtensions {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -524,9 +450,12 @@ impl SaveResults for BadExtensions {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -553,3 +482,22 @@ impl PrintResults for BadExtensions {
}
}
}
impl BadExtensions {
pub const fn get_bad_extensions_files(&self) -> &Vec<BadFileEntry> {
&self.bad_extensions_files
}
pub const fn get_information(&self) -> &Info {
&self.information
}
}
impl CommonData for BadExtensions {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}

188
czkawka_core/src/big_file.rs
View file

@ -3,21 +3,18 @@ use std::fs;
use std::fs::{DirEntry, File, Metadata};
use std::io::{BufWriter, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use humansize::{format_size, BINARY};
use log::{debug, info};
use rayon::prelude::*;
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, split_path};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::{DebugPrint, PrintResults, SaveResults};
#[derive(Clone)]
@ -45,112 +42,57 @@ pub struct Info {
pub number_of_real_files: usize,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
/// Struct with required information's to work
pub struct BigFile {
tool_type: ToolType,
text_messages: Messages,
common_data: CommonToolData,
information: Info,
big_files: Vec<(u64, FileEntry)>,
excluded_items: ExcludedItems,
directories: Directories,
allowed_extensions: Extensions,
recursive_search: bool,
number_of_files_to_check: usize,
delete_method: DeleteMethod,
stopped_search: bool,
search_mode: SearchMode,
}
impl BigFile {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::BigFile,
text_messages: Default::default(),
information: Info::new(),
common_data: CommonToolData::new(ToolType::BigFile),
information: Info::default(),
big_files: Default::default(),
excluded_items: ExcludedItems::new(),
directories: Directories::new(),
allowed_extensions: Extensions::new(),
recursive_search: true,
number_of_files_to_check: 50,
delete_method: DeleteMethod::None,
stopped_search: false,
search_mode: SearchMode::BiggestFiles,
}
}
pub fn find_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_directories();
info!("Starting finding big files");
let start_time = std::time::Instant::now();
self.find_big_files_internal(stop_receiver, progress_sender);
info!("Ended finding big files which took {:?}", start_time.elapsed());
}
fn find_big_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
if !self.look_for_big_files(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
self.delete_files();
self.debug_print();
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
pub fn set_search_mode(&mut self, search_mode: SearchMode) {
self.search_mode = search_mode;
}
#[must_use]
pub const fn get_big_files(&self) -> &Vec<(u64, FileEntry)> {
&self.big_files
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
/// List of allowed extensions, only files with this extensions will be checking if are duplicates
pub fn set_allowed_extensions(&mut self, allowed_extensions: String) {
self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages);
}
fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("look_for_big_files - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
let mut old_map: BTreeMap<u64, Vec<FileEntry>> = Default::default();
// Add root folders for finding
for id in &self.directories.included_directories {
for id in &self.common_data.directories.included_directories {
folders_to_check.push(id.clone());
}
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None, self.common_data.tool_type);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -181,9 +123,9 @@ impl BigFile {
&mut warnings,
current_folder,
entry_data,
self.recursive_search,
&self.directories,
&self.excluded_items,
self.common_data.recursive_search,
&self.common_data.directories,
&self.common_data.excluded_items,
);
} else if metadata.is_file() {
self.collect_file_entry(&atomic_counter, &metadata, entry_data, &mut fe_result, &mut warnings, current_folder);
@ -199,9 +141,9 @@ impl BigFile {
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
for (size, fe) in fe_result {
old_map.entry(size).or_insert_with(Vec::new).push(fe);
old_map.entry(size).or_default().push(fe);
}
}
}
@ -210,6 +152,7 @@ impl BigFile {
self.extract_n_biggest_files(old_map);
debug!("look_for_big_files - end");
true
}
@ -232,12 +175,12 @@ impl BigFile {
return;
};
if !self.allowed_extensions.matches_filename(&file_name_lowercase) {
if !self.common_data.allowed_extensions.matches_filename(&file_name_lowercase) {
return;
}
let current_file_name = current_folder.join(entry_data.file_name());
if self.excluded_items.is_excluded(&current_file_name) {
if self.common_data.excluded_items.is_excluded(&current_file_name) {
return;
}
@ -251,6 +194,7 @@ impl BigFile {
}
pub fn extract_n_biggest_files(&mut self, old_map: BTreeMap<u64, Vec<FileEntry>>) {
debug!("extract_n_biggest_files - start");
let iter: Box<dyn Iterator<Item = _>>;
if self.search_mode == SearchMode::SmallestFiles {
iter = Box::new(old_map.into_iter());
@ -278,28 +222,7 @@ impl BigFile {
break;
}
}
}
pub fn set_number_of_files_to_check(&mut self, number_of_files_to_check: usize) {
self.number_of_files_to_check = number_of_files_to_check;
}
/// Setting excluded items which needs to contains * wildcard
/// Are a lot of slower than absolute path, so it should be used to heavy
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
fn optimize_directories(&mut self) {
self.directories.optimize_directories(self.recursive_search, &mut self.text_messages);
}
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) {
self.directories.set_included_directory(included_directory, &mut self.text_messages);
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
debug!("extract_n_biggest_files - end");
}
/// Function to delete files, from filed Vector
@ -308,7 +231,7 @@ impl BigFile {
DeleteMethod::Delete => {
for (_, file_entry) in &self.big_files {
if fs::remove_file(&file_entry.path).is_err() {
self.text_messages.warnings.push(file_entry.path.display().to_string());
self.common_data.text_messages.warnings.push(file_entry.path.display().to_string());
}
}
}
@ -334,22 +257,11 @@ impl DebugPrint for BigFile {
{
return;
}
println!("---------------DEBUG PRINT---------------");
println!("### Information's");
println!("Errors size - {}", self.text_messages.errors.len());
println!("Warnings size - {}", self.text_messages.warnings.len());
println!("Messages size - {}", self.text_messages.messages.len());
println!("### Other");
println!("### INDIVIDUAL DEBUG PRINT ###");
println!("Big files size {} in {} groups", self.information.number_of_real_files, self.big_files.len());
println!("Excluded items - {:?}", self.excluded_items.items);
println!("Included directories - {:?}", self.directories.included_directories);
println!("Excluded directories - {:?}", self.directories.excluded_directories);
println!("Recursive search - {}", self.recursive_search);
#[cfg(target_family = "unix")]
println!("Skip other filesystems - {}", self.directories.exclude_other_filesystems());
println!("Number of files to check - {:?}", self.number_of_files_to_check);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -365,7 +277,7 @@ impl SaveResults for BigFile {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -374,9 +286,12 @@ impl SaveResults for BigFile {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -409,3 +324,34 @@ impl PrintResults for BigFile {
}
}
}
impl CommonData for BigFile {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}
impl BigFile {
pub fn set_search_mode(&mut self, search_mode: SearchMode) {
self.search_mode = search_mode;
}
pub const fn get_big_files(&self) -> &Vec<(u64, FileEntry)> {
&self.big_files
}
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
pub fn set_number_of_files_to_check(&mut self, number_of_files_to_check: usize) {
self.number_of_files_to_check = number_of_files_to_check;
}
}

197
czkawka_core/src/broken_files.rs
View file

@ -5,11 +5,11 @@ use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::{fs, mem, panic};
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use log::{debug, info};
use pdf::file::FileOptions;
use pdf::object::ParseOptions;
use pdf::PdfError;
@ -22,10 +22,8 @@ use crate::common::{
IMAGE_RS_BROKEN_FILES_EXTENSIONS, PDF_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS,
};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::*;
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
@ -70,74 +68,48 @@ pub struct Info {
pub number_of_broken_files: usize,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
pub struct BrokenFiles {
tool_type: ToolType,
text_messages: Messages,
common_data: CommonToolData,
information: Info,
files_to_check: BTreeMap<String, FileEntry>,
broken_files: Vec<FileEntry>,
directories: Directories,
allowed_extensions: Extensions,
excluded_items: ExcludedItems,
recursive_search: bool,
delete_method: DeleteMethod,
stopped_search: bool,
checked_types: CheckedTypes,
use_cache: bool,
// TODO add this to GUI
delete_outdated_cache: bool,
save_also_as_json: bool,
}
impl BrokenFiles {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::BrokenFiles,
text_messages: Messages::new(),
information: Info::new(),
recursive_search: true,
allowed_extensions: Extensions::new(),
directories: Directories::new(),
excluded_items: ExcludedItems::new(),
common_data: CommonToolData::new(ToolType::BrokenFiles),
information: Info::default(),
files_to_check: Default::default(),
delete_method: DeleteMethod::None,
stopped_search: false,
broken_files: Default::default(),
use_cache: true,
delete_outdated_cache: true,
save_also_as_json: false,
checked_types: CheckedTypes::PDF | CheckedTypes::AUDIO | CheckedTypes::IMAGE | CheckedTypes::ARCHIVE,
}
}
pub fn find_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.directories.optimize_directories(self.recursive_search, &mut self.text_messages);
info!("Starting finding broken files");
let start_time = std::time::Instant::now();
self.find_broken_files_internal(stop_receiver, progress_sender);
info!("Ended finding broken files which took {:?}", start_time.elapsed());
}
pub fn find_broken_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
if !self.look_for_broken_files(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
self.delete_files();
self.debug_print();
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
#[must_use]
pub const fn get_broken_files(&self) -> &Vec<FileEntry> {
&self.broken_files
}
@ -146,12 +118,6 @@ impl BrokenFiles {
self.checked_types = checked_types;
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
@ -160,50 +126,17 @@ impl BrokenFiles {
self.delete_method = delete_method;
}
pub fn set_save_also_as_json(&mut self, save_also_as_json: bool) {
self.save_also_as_json = save_also_as_json;
}
pub fn set_use_cache(&mut self, use_cache: bool) {
self.use_cache = use_cache;
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) -> bool {
self.directories.set_included_directory(included_directory, &mut self.text_messages)
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
pub fn set_allowed_extensions(&mut self, allowed_extensions: String) {
self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages);
}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
// Add root folders for finding
for id in &self.directories.included_directories {
for id in &self.common_data.directories.included_directories {
folders_to_check.push(id.clone());
}
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None, self.common_data.tool_type);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -234,9 +167,9 @@ impl BrokenFiles {
&mut warnings,
current_folder,
entry_data,
self.recursive_search,
&self.directories,
&self.excluded_items,
self.common_data.recursive_search,
&self.common_data.directories,
&self.common_data.excluded_items,
);
} else if metadata.is_file() {
if let Some(file_entry) = self.get_file_entry(&metadata, &atomic_counter, entry_data, &mut warnings, current_folder) {
@ -254,7 +187,7 @@ impl BrokenFiles {
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
for (name, fe) in fe_result {
self.files_to_check.insert(name, fe);
}
@ -263,6 +196,7 @@ impl BrokenFiles {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug!("check_files - end");
true
}
fn get_file_entry(
@ -279,7 +213,7 @@ impl BrokenFiles {
return None;
};
if !self.allowed_extensions.matches_filename(&file_name_lowercase) {
if !self.common_data.allowed_extensions.matches_filename(&file_name_lowercase) {
return None;
}
@ -293,7 +227,7 @@ impl BrokenFiles {
}
let current_file_name = current_folder.join(entry_data.file_name());
if self.excluded_items.is_excluded(&current_file_name) {
if self.common_data.excluded_items.is_excluded(&current_file_name) {
return None;
}
@ -406,15 +340,16 @@ impl BrokenFiles {
}
}
fn look_for_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
fn load_cache(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) {
debug!("load_cache - start (using cache {})", self.common_data.use_cache);
let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default();
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default();
let files_to_check = mem::take(&mut self.files_to_check);
if self.use_cache {
loaded_hash_map = match load_cache_from_file(&mut self.text_messages, self.delete_outdated_cache) {
if self.common_data.use_cache {
loaded_hash_map = match load_cache_from_file(&mut self.common_data.text_messages, self.common_data.delete_outdated_cache) {
Some(t) => t,
None => Default::default(),
};
@ -440,9 +375,16 @@ impl BrokenFiles {
loaded_hash_map = Default::default();
non_cached_files_to_check = files_to_check;
}
debug!("load_cache - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
fn look_for_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("look_for_broken_files - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache();
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type);
let mut vec_file_entry: Vec<FileEntry> = non_cached_files_to_check
.into_par_iter()
@ -471,18 +413,7 @@ impl BrokenFiles {
// Just connect loaded results with already calculated
vec_file_entry.extend(records_already_cached.into_values());
if self.use_cache {
// Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = Default::default();
for file_entry in vec_file_entry.clone() {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
for (_name, file_entry) in loaded_hash_map {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
save_cache_to_file(&all_results, &mut self.text_messages, self.save_also_as_json);
}
self.save_to_cache(&vec_file_entry, loaded_hash_map);
self.broken_files = vec_file_entry
.into_par_iter()
@ -494,8 +425,25 @@ impl BrokenFiles {
// Clean unused data
self.files_to_check = Default::default();
debug!("look_for_broken_files - end");
true
}
fn save_to_cache(&mut self, vec_file_entry: &[FileEntry], loaded_hash_map: BTreeMap<String, FileEntry>) {
debug!("save_to_cache - start, using cache {}", self.common_data.use_cache);
if self.common_data.use_cache {
// Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = Default::default();
for file_entry in vec_file_entry.iter().cloned() {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
for (_name, file_entry) in loaded_hash_map {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
save_cache_to_file(&all_results, &mut self.common_data.text_messages, self.common_data.save_also_as_json);
}
debug!("save_to_cache - end");
}
/// Function to delete files, from filed Vector
fn delete_files(&mut self) {
@ -503,7 +451,7 @@ impl BrokenFiles {
DeleteMethod::Delete => {
for file_entry in &self.broken_files {
if fs::remove_file(&file_entry.path).is_err() {
self.text_messages.warnings.push(file_entry.path.display().to_string());
self.common_data.text_messages.warnings.push(file_entry.path.display().to_string());
}
}
}
@ -530,21 +478,8 @@ impl DebugPrint for BrokenFiles {
return;
}
println!("---------------DEBUG PRINT---------------");
println!("### Information's");
println!("Errors size - {}", self.text_messages.errors.len());
println!("Warnings size - {}", self.text_messages.warnings.len());
println!("Messages size - {}", self.text_messages.messages.len());
println!("### Other");
println!("Excluded items - {:?}", self.excluded_items.items);
println!("Included directories - {:?}", self.directories.included_directories);
println!("Excluded directories - {:?}", self.directories.excluded_directories);
println!("Recursive search - {}", self.recursive_search);
#[cfg(target_family = "unix")]
println!("Skip other filesystems - {}", self.directories.exclude_other_filesystems());
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -559,7 +494,7 @@ impl SaveResults for BrokenFiles {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -568,9 +503,12 @@ impl SaveResults for BrokenFiles {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -724,3 +662,12 @@ fn validate_pdf_error(file_entry: &mut FileEntry, e: PdfError) -> PdfError {
file_entry.error_string = error_string;
unpack_pdf_error(e)
}
impl CommonData for BrokenFiles {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}

70
czkawka_core/src/common.rs
View file

@ -12,10 +12,12 @@ use std::{fs, thread};
use anyhow::Result;
use directories_next::ProjectDirs;
use futures::channel::mpsc::UnboundedSender;
use handsome_logger::{ColorChoice, ConfigBuilder, TerminalMode};
use image::{DynamicImage, ImageBuffer, Rgb};
use imagepipe::{ImageSource, Pipeline};
#[cfg(feature = "heif")]
use libheif_rs::{ColorSpace, HeifContext, RgbChroma};
use log::{debug, LevelFilter, Record};
// #[cfg(feature = "heif")]
// use libheif_rs::LibHeif;
@ -35,11 +37,25 @@ pub fn get_number_of_threads() -> usize {
}
}
fn filtering_messages(record: &Record) -> bool {
if let Some(module_path) = record.module_path() {
!["symphonia", "i18n_embed"].iter().any(|&x| module_path.contains(x))
} else {
true
}
}
pub fn setup_logger(disabled_printing: bool) {
let log_level = if disabled_printing { LevelFilter::Off } else { LevelFilter::Info };
let config = ConfigBuilder::default().set_level(log_level).set_message_filtering(Some(filtering_messages)).build();
handsome_logger::TermLogger::init(config, TerminalMode::Mixed, ColorChoice::Always).unwrap();
}
pub fn set_default_number_of_threads() {
set_number_of_threads(num_cpus::get());
}
#[must_use]
pub fn get_default_number_of_threads() -> usize {
num_cpus::get()
}
@ -66,7 +82,7 @@ pub const IMAGE_RS_BROKEN_FILES_EXTENSIONS: &[&str] = &[
];
pub const HEIC_EXTENSIONS: &[&str] = &[".heif", ".heifs", ".heic", ".heics", ".avci", ".avcs", ".avif", ".avifs"];
pub const ZIP_FILES_EXTENSIONS: &[&str] = &[".zip"];
pub const ZIP_FILES_EXTENSIONS: &[&str] = &[".zip", ".jar"];
pub const PDF_FILES_EXTENSIONS: &[&str] = &[".pdf"];
@ -78,7 +94,8 @@ pub const VIDEO_FILES_EXTENSIONS: &[&str] = &[
".mp4", ".mpv", ".flv", ".mp4a", ".webm", ".mpg", ".mp2", ".mpeg", ".m4p", ".m4v", ".avi", ".wmv", ".qt", ".mov", ".swf", ".mkv",
];
pub const LOOP_DURATION: u32 = 200; //ms
pub const LOOP_DURATION: u32 = 20; //ms
pub const SEND_PROGRESS_DATA_TIME_BETWEEN: u32 = 200; //ms
pub struct Common();
@ -195,7 +212,6 @@ pub fn get_dynamic_image_from_raw_image(path: impl AsRef<Path> + std::fmt::Debug
Some(DynamicImage::ImageRgb8(image))
}
#[must_use]
pub fn split_path(path: &Path) -> (String, String) {
match (path.parent(), path.file_name()) {
(Some(dir), Some(file)) => (dir.display().to_string(), file.to_string_lossy().into_owned()),
@ -204,7 +220,6 @@ pub fn split_path(path: &Path) -> (String, String) {
}
}
#[must_use]
pub fn create_crash_message(library_name: &str, file_path: &str, home_library_url: &str) -> String {
format!("{library_name} library crashed when opening \"{file_path}\", please check if this is fixed with the latest version of {library_name} (e.g. with https://github.com/qarmin/crates_tester) and if it is not fixed, please report bug here - {home_library_url}")
}
@ -221,7 +236,6 @@ impl Common {
);
}
#[must_use]
pub fn delete_multiple_entries(entries: &[String]) -> Vec<String> {
let mut path: &Path;
let mut warnings: Vec<String> = Vec::new();
@ -237,7 +251,7 @@ impl Common {
}
warnings
}
#[must_use]
pub fn delete_one_entry(entry: &str) -> String {
let path: &Path = Path::new(entry);
let mut warning: String = String::new();
@ -252,7 +266,7 @@ impl Common {
}
/// Function to check if directory match expression
#[must_use]
pub fn regex_check(expression: &str, directory: impl AsRef<Path>) -> bool {
if expression == "*" {
return true;
@ -305,7 +319,6 @@ impl Common {
true
}
#[must_use]
pub fn normalize_windows_path(path_to_change: impl AsRef<Path>) -> PathBuf {
let path = path_to_change.as_ref();
@ -365,7 +378,6 @@ pub fn check_folder_children(
dir_result.push(next_folder);
}
#[must_use]
pub fn filter_reference_folders_generic<T>(entries_to_check: Vec<Vec<T>>, directories: &Directories) -> Vec<(T, Vec<T>)>
where
T: ResultEntry,
@ -385,7 +397,6 @@ where
.collect::<Vec<(T, Vec<T>)>>()
}
#[must_use]
pub fn prepare_thread_handler_common(
progress_sender: Option<&UnboundedSender<ProgressData>>,
current_stage: u8,
@ -401,21 +412,28 @@ pub fn prepare_thread_handler_common(
let progress_send = progress_sender.clone();
let progress_thread_run = progress_thread_run.clone();
let atomic_counter = atomic_counter.clone();
thread::spawn(move || loop {
progress_send
.unbounded_send(ProgressData {
checking_method,
current_stage,
max_stage,
entries_checked: atomic_counter.load(Ordering::Relaxed),
entries_to_check: max_value,
tool_type,
})
.unwrap();
if !progress_thread_run.load(Ordering::Relaxed) {
break;
thread::spawn(move || {
let mut time_since_last_send = SystemTime::now();
loop {
if time_since_last_send.elapsed().unwrap().as_millis() > SEND_PROGRESS_DATA_TIME_BETWEEN as u128 {
progress_send
.unbounded_send(ProgressData {
checking_method,
current_stage,
max_stage,
entries_checked: atomic_counter.load(Ordering::Relaxed),
entries_to_check: max_value,
tool_type,
})
.unwrap();
time_since_last_send = SystemTime::now();
}
if !progress_thread_run.load(Ordering::Relaxed) {
break;
}
sleep(Duration::from_millis(LOOP_DURATION as u64));
}
sleep(Duration::from_millis(LOOP_DURATION as u64));
})
} else {
thread::spawn(|| {})
@ -424,8 +442,10 @@ pub fn prepare_thread_handler_common(
}
pub fn send_info_and_wait_for_ending_all_threads(progress_thread_run: &Arc<AtomicBool>, progress_thread_handle: JoinHandle<()>) {
debug!("Sending info to stop all threads");
progress_thread_run.store(false, Ordering::Relaxed);
progress_thread_handle.join().unwrap();
debug!("All threads stopped");
}
#[cfg(test)]

24
czkawka_core/src/common_dir_traversal.rs
View file

@ -27,7 +27,7 @@ pub struct ProgressData {
pub tool_type: ToolType,
}
#[derive(PartialEq, Eq, Clone, Debug, Copy)]
#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)]
pub enum ToolType {
Duplicate,
EmptyFolders,
@ -40,11 +40,13 @@ pub enum ToolType {
SimilarImages,
SimilarVideos,
TemporaryFiles,
#[default]
None,
}
#[derive(PartialEq, Eq, Clone, Debug, Copy)]
#[derive(PartialEq, Eq, Clone, Debug, Copy, Default)]
pub enum CheckingMethod {
#[default]
None,
Name,
SizeName,
@ -62,6 +64,7 @@ pub struct FileEntry {
pub hash: String,
pub symlink_info: Option<SymlinkInfo>,
}
impl ResultEntry for FileEntry {
fn get_path(&self) -> &Path {
&self.path
@ -161,7 +164,6 @@ impl<'a, 'b> Default for DirTraversalBuilder<'a, 'b, ()> {
}
impl<'a, 'b> DirTraversalBuilder<'a, 'b, ()> {
#[must_use]
pub fn new() -> DirTraversalBuilder<'a, 'b, ()> {
DirTraversalBuilder {
group_by: None,
@ -183,86 +185,72 @@ impl<'a, 'b> DirTraversalBuilder<'a, 'b, ()> {
}
impl<'a, 'b, F> DirTraversalBuilder<'a, 'b, F> {
#[must_use]
pub fn root_dirs(mut self, dirs: Vec<PathBuf>) -> Self {
self.root_dirs = dirs;
self
}
#[must_use]
pub fn stop_receiver(mut self, stop_receiver: Option<&'a Receiver<()>>) -> Self {
self.stop_receiver = stop_receiver;
self
}
#[must_use]
pub fn progress_sender(mut self, progress_sender: Option<&'b UnboundedSender<ProgressData>>) -> Self {
self.progress_sender = progress_sender;
self
}
#[must_use]
pub fn checking_method(mut self, checking_method: CheckingMethod) -> Self {
self.checking_method = checking_method;
self
}
#[must_use]
pub fn max_stage(mut self, max_stage: u8) -> Self {
self.max_stage = max_stage;
self
}
#[must_use]
pub fn minimal_file_size(mut self, minimal_file_size: u64) -> Self {
self.minimal_file_size = Some(minimal_file_size);
self
}
#[must_use]
pub fn maximal_file_size(mut self, maximal_file_size: u64) -> Self {
self.maximal_file_size = Some(maximal_file_size);
self
}
#[must_use]
pub fn collect(mut self, collect: Collect) -> Self {
self.collect = collect;
self
}
#[must_use]
pub fn directories(mut self, directories: Directories) -> Self {
self.directories = Some(directories);
self
}
#[must_use]
pub fn allowed_extensions(mut self, allowed_extensions: Extensions) -> Self {
self.allowed_extensions = Some(allowed_extensions);
self
}
#[must_use]
pub fn excluded_items(mut self, excluded_items: ExcludedItems) -> Self {
self.excluded_items = Some(excluded_items);
self
}
#[must_use]
pub fn recursive_search(mut self, recursive_search: bool) -> Self {
self.recursive_search = recursive_search;
self
}
#[must_use]
pub fn tool_type(mut self, tool_type: ToolType) -> Self {
self.tool_type = tool_type;
self
}
#[cfg(target_family = "unix")]
#[must_use]
pub fn exclude_other_filesystems(mut self, exclude_other_filesystems: bool) -> Self {
match self.directories {
Some(ref mut directories) => directories.set_exclude_other_filesystems(exclude_other_filesystems),
@ -489,7 +477,7 @@ where
all_warnings.extend(warnings);
for fe in fe_result {
let key = (self.group_by)(&fe);
grouped_file_entries.entry(key).or_insert_with(Vec::new).push(fe);
grouped_file_entries.entry(key).or_default().push(fe);
}
for current_folder in &set_as_not_empty_folder_list {
set_as_not_empty_folder(&mut folder_entries, current_folder);

67
czkawka_core/src/common_directory.rs
View file

@ -1,14 +1,12 @@
use std::path::{Path, PathBuf};
#[cfg(target_family = "unix")]
use std::{fs, os::unix::fs::MetadataExt};
use crate::common::Common;
use crate::common_messages::Messages;
use crate::flc;
use crate::localizer_core::generate_translation_hashmap;
#[derive(Clone, Default)]
#[derive(Debug, Clone, Default)]
pub struct Directories {
pub excluded_directories: Vec<PathBuf>,
pub included_directories: Vec<PathBuf>,
@ -19,7 +17,6 @@ pub struct Directories {
}
impl Directories {
#[must_use]
pub fn new() -> Self {
Default::default()
}
@ -29,10 +26,14 @@ impl Directories {
}
/// Setting included directories, at least one must be provided or scan won't start
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>, text_messages: &mut Messages) -> bool {
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) -> (Vec<String>, Vec<String>, Vec<String>) {
let messages: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
let mut warnings: Vec<String> = Vec::new();
if included_directory.is_empty() {
text_messages.errors.push(flc!("core_missing_no_chosen_included_directory"));
return false;
errors.push(flc!("core_missing_no_chosen_included_directory"));
return (messages, warnings, errors);
}
let directories: Vec<PathBuf> = included_directory;
@ -40,7 +41,7 @@ impl Directories {
let mut checked_directories: Vec<PathBuf> = Vec::new();
for directory in directories {
if directory.to_string_lossy().contains('*') {
text_messages.warnings.push(flc!(
warnings.push(flc!(
"core_directory_wildcard_no_supported",
generate_translation_hashmap(vec![("path", directory.display().to_string())])
));
@ -49,7 +50,7 @@ impl Directories {
#[cfg(not(target_family = "windows"))]
if directory.is_relative() {
text_messages.warnings.push(flc!(
warnings.push(flc!(
"core_directory_relative_path",
generate_translation_hashmap(vec![("path", directory.display().to_string())])
));
@ -57,7 +58,7 @@ impl Directories {
}
#[cfg(target_family = "windows")]
if directory.is_relative() && !directory.starts_with("\\") {
text_messages.warnings.push(flc!(
warnings.push(flc!(
"core_directory_relative_path",
generate_translation_hashmap(vec![("path", directory.display().to_string())])
));
@ -65,14 +66,14 @@ impl Directories {
}
if !directory.exists() {
text_messages.warnings.push(flc!(
warnings.push(flc!(
"core_directory_must_exists",
generate_translation_hashmap(vec![("path", directory.display().to_string())])
));
continue;
}
if !directory.is_dir() {
text_messages.warnings.push(flc!(
warnings.push(flc!(
"core_directory_must_be_directory",
generate_translation_hashmap(vec![("path", directory.display().to_string())])
));
@ -82,19 +83,23 @@ impl Directories {
}
if checked_directories.is_empty() {
text_messages.warnings.push(flc!("core_included_directory_zero_valid_directories"));
return false;
warnings.push(flc!("core_included_directory_zero_valid_directories"));
return (messages, warnings, errors);
}
self.included_directories = checked_directories;
true
(messages, warnings, errors)
}
/// Setting absolute path to exclude from search
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>, text_messages: &mut Messages) {
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) -> (Vec<String>, Vec<String>, Vec<String>) {
let messages: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
let mut warnings: Vec<String> = Vec::new();
if excluded_directory.is_empty() {
return;
return (messages, warnings, errors);
}
let directories: Vec<PathBuf> = excluded_directory;
@ -103,11 +108,11 @@ impl Directories {
for directory in directories {
let directory_as_string = directory.to_string_lossy();
if directory_as_string == "/" {
text_messages.errors.push(flc!("core_excluded_directory_pointless_slash"));
errors.push(flc!("core_excluded_directory_pointless_slash"));
break;
}
if directory_as_string.contains('*') {
text_messages.warnings.push(flc!(
warnings.push(flc!(
"core_directory_wildcard_no_supported",
generate_translation_hashmap(vec![("path", directory.display().to_string())])
));
@ -115,7 +120,7 @@ impl Directories {
}
#[cfg(not(target_family = "windows"))]
if directory.is_relative() {
text_messages.warnings.push(flc!(
warnings.push(flc!(
"core_directory_relative_path",
generate_translation_hashmap(vec![("path", directory.display().to_string())])
));
@ -123,7 +128,7 @@ impl Directories {
}
#[cfg(target_family = "windows")]
if directory.is_relative() && !directory.starts_with("\\") {
text_messages.warnings.push(flc!(
warnings.push(flc!(
"core_directory_relative_path",
generate_translation_hashmap(vec![("path", directory.display().to_string())])
));
@ -135,7 +140,7 @@ impl Directories {
continue;
}
if !directory.is_dir() {
text_messages.warnings.push(flc!(
warnings.push(flc!(
"core_directory_must_be_directory",
generate_translation_hashmap(vec![("path", directory.display().to_string())])
));
@ -144,6 +149,8 @@ impl Directories {
checked_directories.push(directory);
}
self.excluded_directories = checked_directories;
(messages, warnings, errors)
}
#[cfg(target_family = "unix")]
@ -152,7 +159,11 @@ impl Directories {
}
/// Remove unused entries when included or excluded overlaps with each other or are duplicated etc.
pub fn optimize_directories(&mut self, recursive_search: bool, text_messages: &mut Messages) -> bool {
pub fn optimize_directories(&mut self, recursive_search: bool) -> (Vec<String>, Vec<String>, Vec<String>) {
let messages: Vec<String> = Vec::new();
let mut errors: Vec<String> = Vec::new();
let warnings: Vec<String> = Vec::new();
let mut optimized_included: Vec<PathBuf> = Vec::new();
let mut optimized_excluded: Vec<PathBuf> = Vec::new();
@ -280,8 +291,8 @@ impl Directories {
}
if self.included_directories.is_empty() {
text_messages.errors.push(flc!("core_directory_overlap"));
return false;
errors.push(flc!("core_directory_overlap"));
return (messages, warnings, errors);
}
// Not needed, but better is to have sorted everything
@ -294,7 +305,7 @@ impl Directories {
for d in &self.included_directories {
match fs::metadata(d) {
Ok(m) => self.included_dev_ids.push(m.dev()),
Err(_) => text_messages.errors.push(flc!(
Err(_) => errors.push(flc!(
"core_directory_unable_to_get_device_id",
generate_translation_hashmap(vec![("path", d.display().to_string())])
)),
@ -302,10 +313,9 @@ impl Directories {
}
}
true
(messages, warnings, errors)
}
#[must_use]
pub fn is_in_referenced_directory(&self, path: &Path) -> bool {
self.reference_directories.iter().any(|e| path.starts_with(e))
}
@ -320,7 +330,6 @@ impl Directories {
}
#[cfg(target_family = "unix")]
#[must_use]
pub fn exclude_other_filesystems(&self) -> bool {
self.exclude_other_filesystems.unwrap_or(false)
}

26
czkawka_core/src/common_extensions.rs
View file

@ -1,20 +1,21 @@
use crate::common_messages::Messages;
#[derive(Clone, Default)]
#[derive(Debug, Clone, Default)]
pub struct Extensions {
file_extensions: Vec<String>,
}
impl Extensions {
#[must_use]
pub fn new() -> Self {
Default::default()
}
/// List of allowed extensions, only files with this extensions will be checking if are duplicates
/// After, extensions cannot contains any dot, commas etc.
pub fn set_allowed_extensions(&mut self, mut allowed_extensions: String, text_messages: &mut Messages) {
pub fn set_allowed_extensions(&mut self, mut allowed_extensions: String) -> (Vec<String>, Vec<String>, Vec<String>) {
let mut messages = Vec::new();
let mut warnings = Vec::new();
let errors = Vec::new();
if allowed_extensions.trim().is_empty() {
return;
return (messages, warnings, errors);
}
allowed_extensions = allowed_extensions.replace("IMAGE", "jpg,kra,gif,png,bmp,tiff,hdr,svg");
allowed_extensions = allowed_extensions.replace("VIDEO", "mp4,flv,mkv,webm,vob,ogv,gifv,avi,mov,wmv,mpg,m4v,m4p,mpeg,3gp");
@ -32,14 +33,12 @@ impl Extensions {
}
if extension[1..].contains('.') {
text_messages.warnings.push(format!("{extension} is not valid extension because contains dot inside"));
warnings.push(format!("{extension} is not valid extension because contains dot inside"));
continue;
}
if extension[1..].contains(' ') {
text_messages
.warnings
.push(format!("{extension} is not valid extension because contains empty space inside"));
warnings.push(format!("{extension} is not valid extension because contains empty space inside"));
continue;
}
@ -49,13 +48,11 @@ impl Extensions {
}
if self.file_extensions.is_empty() {
text_messages
.messages
.push("No valid extensions were provided, so allowing all extensions by default.".to_string());
messages.push("No valid extensions were provided, so allowing all extensions by default.".to_string());
}
(messages, warnings, errors)
}
#[must_use]
pub fn matches_filename(&self, file_name: &str) -> bool {
// assert_eq!(file_name, file_name.to_lowercase());
if !self.file_extensions.is_empty() && !self.file_extensions.iter().any(|e| file_name.ends_with(e)) {
@ -64,7 +61,6 @@ impl Extensions {
true
}
#[must_use]
pub fn using_custom_extensions(&self) -> bool {
!self.file_extensions.is_empty()
}

16
czkawka_core/src/common_items.rs
View file

@ -1,23 +1,24 @@
use std::path::Path;
use crate::common::Common;
use crate::common_messages::Messages;
#[derive(Clone, Default)]
#[derive(Debug, Clone, Default)]
pub struct ExcludedItems {
pub items: Vec<String>,
}
impl ExcludedItems {
#[must_use]
pub fn new() -> Self {
Default::default()
}
/// Setting excluded items which needs to contains * wildcard
/// Are a lot of slower than absolute path, so it should be used to heavy
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>, text_messages: &mut Messages) {
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) -> (Vec<String>, Vec<String>, Vec<String>) {
let messages: Vec<String> = Vec::new();
let mut warnings: Vec<String> = Vec::new();
let errors: Vec<String> = Vec::new();
if excluded_items.is_empty() {
return;
return (messages, warnings, errors);
}
let expressions: Vec<String> = excluded_items;
@ -43,15 +44,14 @@ impl ExcludedItems {
continue;
}
if !expression.contains('*') {
text_messages
.warnings
.push("Excluded Items Warning: Wildcard * is required in expression, ignoring ".to_string() + expression.as_str());
warnings.push("Excluded Items Warning: Wildcard * is required in expression, ignoring ".to_string() + expression.as_str());
continue;
}
checked_expressions.push(expression);
}
self.items = checked_expressions;
(messages, warnings, errors)
}
/// Checks whether a specified path is excluded from searching

11
czkawka_core/src/common_messages.rs
View file

@ -1,4 +1,4 @@
#[derive(Default)]
#[derive(Debug, Default, Clone)]
pub struct Messages {
pub messages: Vec<String>,
pub warnings: Vec<String>,
@ -6,14 +6,13 @@ pub struct Messages {
}
impl Messages {
#[must_use]
pub fn new() -> Self {
Default::default()
}
pub fn print_messages(&self) {
println!("{}", self.create_messages_text());
}
#[must_use]
pub fn create_messages_text(&self) -> String {
let mut text_to_return: String = String::new();
@ -48,4 +47,10 @@ impl Messages {
text_to_return
}
pub fn extend_messages_with(&mut self, messages: Vec<String>, warnings: Vec<String>, errors: Vec<String>) {
self.messages.extend(messages);
self.warnings.extend(warnings);
self.errors.extend(errors);
}
}

178
czkawka_core/src/common_tool.rs Normal file
View file

@ -0,0 +1,178 @@
use std::path::PathBuf;
use crate::common_dir_traversal::ToolType;
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
#[derive(Debug, Clone, Default)]
pub struct CommonToolData {
pub(crate) tool_type: ToolType,
pub(crate) text_messages: Messages,
pub(crate) directories: Directories,
pub(crate) allowed_extensions: Extensions,
pub(crate) excluded_items: ExcludedItems,
pub(crate) recursive_search: bool,
// delete_method: DeleteMethod, // ?
pub(crate) maximal_file_size: u64,
pub(crate) minimal_file_size: u64,
pub(crate) stopped_search: bool,
pub(crate) use_cache: bool,
pub(crate) delete_outdated_cache: bool,
pub(crate) save_also_as_json: bool,
pub(crate) use_reference_folders: bool,
}
impl CommonToolData {
pub fn new(tool_type: ToolType) -> Self {
Self {
tool_type,
text_messages: Messages::new(),
directories: Directories::new(),
allowed_extensions: Extensions::new(),
excluded_items: ExcludedItems::new(),
recursive_search: true,
maximal_file_size: u64::MAX,
minimal_file_size: 8192,
stopped_search: false,
use_cache: true,
delete_outdated_cache: true,
save_also_as_json: false,
use_reference_folders: false,
}
}
}
pub trait CommonData {
fn get_cd(&self) -> &CommonToolData;
fn get_cd_mut(&mut self) -> &mut CommonToolData;
fn set_use_cache(&mut self, use_cache: bool) {
self.get_cd_mut().use_cache = use_cache;
}
fn get_use_cache(&self) -> bool {
self.get_cd().use_cache
}
fn set_delete_outdated_cache(&mut self, delete_outdated_cache: bool) {
self.get_cd_mut().delete_outdated_cache = delete_outdated_cache;
}
fn get_delete_outdated_cache(&self) -> bool {
self.get_cd().delete_outdated_cache
}
fn get_stopped_search(&self) -> bool {
self.get_cd().stopped_search
}
fn set_stopped_search(&mut self, stopped_search: bool) {
self.get_cd_mut().stopped_search = stopped_search;
}
fn set_maximal_file_size(&mut self, maximal_file_size: u64) {
self.get_cd_mut().maximal_file_size = match maximal_file_size {
0 => 1,
t => t,
};
}
fn get_maximal_file_size(&self) -> u64 {
self.get_cd().maximal_file_size
}
fn set_minimal_file_size(&mut self, minimal_file_size: u64) {
self.get_cd_mut().minimal_file_size = match minimal_file_size {
0 => 1,
t => t,
};
}
fn get_minimal_file_size(&self) -> u64 {
self.get_cd().minimal_file_size
}
fn set_reference_directory(&mut self, reference_directory: Vec<PathBuf>) {
self.get_cd_mut().directories.set_reference_directory(reference_directory);
}
#[cfg(target_family = "unix")]
fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.get_cd_mut().directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
fn get_text_messages(&self) -> &Messages {
&self.get_cd().text_messages
}
fn set_save_also_as_json(&mut self, save_also_as_json: bool) {
self.get_cd_mut().save_also_as_json = save_also_as_json;
}
fn get_save_also_as_json(&self) -> bool {
self.get_cd().save_also_as_json
}
fn set_recursive_search(&mut self, recursive_search: bool) {
self.get_cd_mut().recursive_search = recursive_search;
}
fn get_recursive_search(&self) -> bool {
self.get_cd().recursive_search
}
fn set_use_reference_folders(&mut self, use_reference_folders: bool) {
self.get_cd_mut().use_reference_folders = use_reference_folders;
}
fn get_use_reference_folders(&self) -> bool {
self.get_cd().use_reference_folders
}
fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) {
let (messages, warnings, errors) = self.get_cd_mut().directories.set_included_directory(included_directory);
self.get_cd_mut().text_messages.extend_messages_with(messages, warnings, errors);
}
fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
let (messages, warnings, errors) = self.get_cd_mut().directories.set_excluded_directory(excluded_directory);
self.get_cd_mut().text_messages.messages.extend(messages);
self.get_cd_mut().text_messages.warnings.extend(warnings);
self.get_cd_mut().text_messages.errors.extend(errors);
}
fn set_allowed_extensions(&mut self, allowed_extensions: String) {
let (messages, warnings, errors) = self.get_cd_mut().allowed_extensions.set_allowed_extensions(allowed_extensions);
self.get_cd_mut().text_messages.messages.extend(messages);
self.get_cd_mut().text_messages.warnings.extend(warnings);
self.get_cd_mut().text_messages.errors.extend(errors);
}
fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
let (messages, warnings, errors) = self.get_cd_mut().excluded_items.set_excluded_items(excluded_items);
self.get_cd_mut().text_messages.messages.extend(messages);
self.get_cd_mut().text_messages.warnings.extend(warnings);
self.get_cd_mut().text_messages.errors.extend(errors);
}
fn optimize_dirs_before_start(&mut self) {
let recursive_search = self.get_cd().recursive_search;
let (messages, warnings, errors) = self.get_cd_mut().directories.optimize_directories(recursive_search);
self.get_cd_mut().text_messages.extend_messages_with(messages, warnings, errors);
}
fn debug_print_common(&self) {
println!("---------------DEBUG PRINT COMMON---------------");
println!("Tool type: {:?}", self.get_cd().tool_type);
println!("Directories: {:?}", self.get_cd().directories);
println!("Allowed extensions: {:?}", self.get_cd().allowed_extensions);
println!("Excluded items: {:?}", self.get_cd().excluded_items);
println!("Recursive search: {:?}", self.get_cd().recursive_search);
println!("Maximal file size: {:?}", self.get_cd().maximal_file_size);
println!("Minimal file size: {:?}", self.get_cd().minimal_file_size);
println!("Stopped search: {:?}", self.get_cd().stopped_search);
println!("Use cache: {:?}", self.get_cd().use_cache);
println!("Delete outdated cache: {:?}", self.get_cd().delete_outdated_cache);
println!("Save also as json: {:?}", self.get_cd().save_also_as_json);
println!("---------------DEBUG PRINT MESSAGES---------------");
println!("Errors size - {}", self.get_cd().text_messages.errors.len());
println!("Warnings size - {}", self.get_cd().text_messages.warnings.len());
println!("Messages size - {}", self.get_cd().text_messages.messages.len());
}
}

409
czkawka_core/src/duplicate.rs
View file

@ -9,29 +9,28 @@ use std::io::{self, BufReader, BufWriter, Error, ErrorKind};
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering;
use std::{fs, mem};
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use humansize::{format_size, BINARY};
use log::{debug, info};
use rayon::prelude::*;
use xxhash_rust::xxh3::Xxh3;
use crate::common::{open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::*;
use crate::flc;
use crate::localizer_core::generate_translation_hashmap;
const TEMP_HARDLINK_FILE: &str = "rzeczek.rxrxrxl";
#[derive(PartialEq, Eq, Clone, Debug, Copy)]
#[derive(PartialEq, Eq, Clone, Debug, Copy, Default)]
pub enum HashType {
#[default]
Blake3,
Crc32,
Xxh3,
@ -47,8 +46,9 @@ impl HashType {
}
}
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
#[derive(Eq, PartialEq, Clone, Debug, Copy, Default)]
pub enum DeleteMethod {
#[default]
None,
AllExceptNewest,
AllExceptOldest,
@ -71,53 +71,41 @@ pub struct Info {
pub lost_space_by_hash: u64,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
pub struct DuplicateFinder {
tool_type: ToolType,
text_messages: Messages,
common_data: CommonToolData,
information: Info,
files_with_identical_names: BTreeMap<String, Vec<FileEntry>>, // File Size, File Entry
files_with_identical_size_names: BTreeMap<(u64, String), Vec<FileEntry>>, // File (Size, Name), File Entry
files_with_identical_size: BTreeMap<u64, Vec<FileEntry>>, // File Size, File Entry
files_with_identical_hashes: BTreeMap<u64, Vec<Vec<FileEntry>>>, // File Size, next grouped by file size, next grouped by hash
files_with_identical_names_referenced: BTreeMap<String, (FileEntry, Vec<FileEntry>)>, // File Size, File Entry
files_with_identical_size_names_referenced: BTreeMap<(u64, String), (FileEntry, Vec<FileEntry>)>, // File (Size, Name), File Entry
files_with_identical_size_referenced: BTreeMap<u64, (FileEntry, Vec<FileEntry>)>, // File Size, File Entry
files_with_identical_hashes_referenced: BTreeMap<u64, Vec<(FileEntry, Vec<FileEntry>)>>, // File Size, next grouped by file size, next grouped by hash
directories: Directories,
allowed_extensions: Extensions,
excluded_items: ExcludedItems,
recursive_search: bool,
minimal_file_size: u64,
maximal_file_size: u64,
// File Size, File Entry
files_with_identical_names: BTreeMap<String, Vec<FileEntry>>,
// File (Size, Name), File Entry
files_with_identical_size_names: BTreeMap<(u64, String), Vec<FileEntry>>,
// File Size, File Entry
files_with_identical_size: BTreeMap<u64, Vec<FileEntry>>,
// File Size, next grouped by file size, next grouped by hash
files_with_identical_hashes: BTreeMap<u64, Vec<Vec<FileEntry>>>,
// File Size, File Entry
files_with_identical_names_referenced: BTreeMap<String, (FileEntry, Vec<FileEntry>)>,
// File (Size, Name), File Entry
files_with_identical_size_names_referenced: BTreeMap<(u64, String), (FileEntry, Vec<FileEntry>)>,
// File Size, File Entry
files_with_identical_size_referenced: BTreeMap<u64, (FileEntry, Vec<FileEntry>)>,
// File Size, next grouped by file size, next grouped by hash
files_with_identical_hashes_referenced: BTreeMap<u64, Vec<(FileEntry, Vec<FileEntry>)>>,
check_method: CheckingMethod,
delete_method: DeleteMethod,
hash_type: HashType,
ignore_hard_links: bool,
dryrun: bool,
stopped_search: bool,
use_cache: bool,
use_prehash_cache: bool,
minimal_cache_file_size: u64,
minimal_prehash_cache_file_size: u64,
delete_outdated_cache: bool,
use_reference_folders: bool,
case_sensitive_name_comparison: bool,
}
impl DuplicateFinder {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::Duplicate,
text_messages: Messages::new(),
information: Info::new(),
common_data: CommonToolData::new(ToolType::Duplicate),
information: Info::default(),
files_with_identical_names: Default::default(),
files_with_identical_size: Default::default(),
files_with_identical_size_names: Default::default(),
@ -126,58 +114,55 @@ impl DuplicateFinder {
files_with_identical_size_names_referenced: Default::default(),
files_with_identical_size_referenced: Default::default(),
files_with_identical_hashes_referenced: Default::default(),
recursive_search: true,
allowed_extensions: Extensions::new(),
check_method: CheckingMethod::None,
delete_method: DeleteMethod::None,
minimal_file_size: 8192,
maximal_file_size: u64::MAX,
directories: Directories::new(),
excluded_items: ExcludedItems::new(),
stopped_search: false,
ignore_hard_links: true,
hash_type: HashType::Blake3,
dryrun: false,
use_cache: true,
use_prehash_cache: true,
minimal_cache_file_size: 1024 * 1024 / 4, // By default cache only >= 256 KB files
minimal_prehash_cache_file_size: 0,
delete_outdated_cache: true,
use_reference_folders: false,
case_sensitive_name_comparison: false,
}
}
pub fn find_duplicates(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.directories.optimize_directories(self.recursive_search, &mut self.text_messages);
self.use_reference_folders = !self.directories.reference_directories.is_empty();
info!("Starting finding duplicates");
let start_time = std::time::Instant::now();
self.find_duplicates_internal(stop_receiver, progress_sender);
info!("Ended finding duplicates which took {:?}", start_time.elapsed());
}
fn find_duplicates_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
self.common_data.use_reference_folders = !self.common_data.directories.reference_directories.is_empty();
match self.check_method {
CheckingMethod::Name => {
self.stopped_search = !self.check_files_name(stop_receiver, progress_sender); // TODO restore this to name
if self.stopped_search {
self.common_data.stopped_search = !self.check_files_name(stop_receiver, progress_sender); // TODO restore this to name
if self.common_data.stopped_search {
return;
}
}
CheckingMethod::SizeName => {
self.stopped_search = !self.check_files_size_name(stop_receiver, progress_sender);
if self.stopped_search {
self.common_data.stopped_search = !self.check_files_size_name(stop_receiver, progress_sender);
if self.common_data.stopped_search {
return;
}
}
CheckingMethod::Size => {
self.stopped_search = !self.check_files_size(stop_receiver, progress_sender);
if self.stopped_search {
self.common_data.stopped_search = !self.check_files_size(stop_receiver, progress_sender);
if self.common_data.stopped_search {
return;
}
}
CheckingMethod::Hash => {
self.stopped_search = !self.check_files_size(stop_receiver, progress_sender);
if self.stopped_search {
self.common_data.stopped_search = !self.check_files_size(stop_receiver, progress_sender);
if self.common_data.stopped_search {
return;
}
self.stopped_search = !self.check_files_hash(stop_receiver, progress_sender);
if self.stopped_search {
self.common_data.stopped_search = !self.check_files_hash(stop_receiver, progress_sender);
if self.common_data.stopped_search {
return;
}
}
@ -187,24 +172,14 @@ impl DuplicateFinder {
self.debug_print();
}
pub fn set_delete_outdated_cache(&mut self, delete_outdated_cache: bool) {
self.delete_outdated_cache = delete_outdated_cache;
}
pub fn set_case_sensitive_name_comparison(&mut self, case_sensitive_name_comparison: bool) {
self.case_sensitive_name_comparison = case_sensitive_name_comparison;
}
#[must_use]
pub const fn get_check_method(&self) -> &CheckingMethod {
&self.check_method
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
pub fn set_minimal_cache_file_size(&mut self, minimal_cache_file_size: u64) {
self.minimal_cache_file_size = minimal_cache_file_size;
}
@ -213,46 +188,26 @@ impl DuplicateFinder {
self.minimal_prehash_cache_file_size = minimal_prehash_cache_file_size;
}
#[must_use]
pub const fn get_files_sorted_by_names(&self) -> &BTreeMap<String, Vec<FileEntry>> {
&self.files_with_identical_names
}
pub fn set_use_cache(&mut self, use_cache: bool) {
self.use_cache = use_cache;
}
pub fn set_use_prehash_cache(&mut self, use_prehash_cache: bool) {
self.use_prehash_cache = use_prehash_cache;
}
#[must_use]
pub const fn get_files_sorted_by_size(&self) -> &BTreeMap<u64, Vec<FileEntry>> {
&self.files_with_identical_size
}
#[must_use]
pub const fn get_files_sorted_by_size_name(&self) -> &BTreeMap<(u64, String), Vec<FileEntry>> {
&self.files_with_identical_size_names
}
#[must_use]
pub const fn get_files_sorted_by_hash(&self) -> &BTreeMap<u64, Vec<Vec<FileEntry>>> {
&self.files_with_identical_hashes
}
pub fn set_maximal_file_size(&mut self, maximal_file_size: u64) {
self.maximal_file_size = match maximal_file_size {
0 => 1,
t => t,
};
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
@ -277,69 +232,28 @@ impl DuplicateFinder {
self.delete_method = delete_method;
}
pub fn set_minimal_file_size(&mut self, minimal_file_size: u64) {
self.minimal_file_size = match minimal_file_size {
0 => 1,
t => t,
};
}
#[must_use]
pub fn get_use_reference(&self) -> bool {
self.use_reference_folders
self.common_data.use_reference_folders
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) {
self.directories.set_included_directory(included_directory, &mut self.text_messages);
}
pub fn set_reference_directory(&mut self, reference_directory: Vec<PathBuf>) {
self.directories.set_reference_directory(reference_directory);
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
pub fn set_allowed_extensions(&mut self, allowed_extensions: String) {
self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages);
}
#[must_use]
pub fn get_files_with_identical_hashes_referenced(&self) -> &BTreeMap<u64, Vec<(FileEntry, Vec<FileEntry>)>> {
&self.files_with_identical_hashes_referenced
}
#[must_use]
pub fn get_files_with_identical_name_referenced(&self) -> &BTreeMap<String, (FileEntry, Vec<FileEntry>)> {
&self.files_with_identical_names_referenced
}
#[must_use]
pub fn get_files_with_identical_size_referenced(&self) -> &BTreeMap<u64, (FileEntry, Vec<FileEntry>)> {
&self.files_with_identical_size_referenced
}
#[must_use]
pub fn get_files_with_identical_size_names_referenced(&self) -> &BTreeMap<(u64, String), (FileEntry, Vec<FileEntry>)> {
&self.files_with_identical_size_names_referenced
}
fn check_files_name(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files_name - starting checking for same names");
let group_by_func = if self.case_sensitive_name_comparison {
|fe: &FileEntry| fe.path.file_name().unwrap().to_string_lossy().to_string()
} else {
@ -347,23 +261,24 @@ impl DuplicateFinder {
};
let result = DirTraversalBuilder::new()
.root_dirs(self.directories.included_directories.clone())
.root_dirs(self.common_data.directories.included_directories.clone())
.group_by(group_by_func)
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.checking_method(CheckingMethod::Name)
.directories(self.directories.clone())
.allowed_extensions(self.allowed_extensions.clone())
.excluded_items(self.excluded_items.clone())
.recursive_search(self.recursive_search)
.minimal_file_size(self.minimal_file_size)
.maximal_file_size(self.maximal_file_size)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.minimal_file_size(self.common_data.minimal_file_size)
.maximal_file_size(self.common_data.maximal_file_size)
.build()
.run();
match result {
debug!("check_files_name - after finding file sizes");
let res = match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
self.files_with_identical_names = grouped_file_entries;
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
// Create new BTreeMap without single size entries(files have not duplicates)
let mut new_map: BTreeMap<String, Vec<FileEntry>> = Default::default();
@ -376,12 +291,13 @@ impl DuplicateFinder {
self.files_with_identical_names = new_map;
// Reference - only use in size, because later hash will be counted differently
if self.use_reference_folders {
if self.common_data.use_reference_folders {
let vec = mem::take(&mut self.files_with_identical_names)
.into_iter()
.filter_map(|(_name, vec_file_entry)| {
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) = vec_file_entry
.into_iter()
.partition(|e| self.common_data.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -402,11 +318,13 @@ impl DuplicateFinder {
unreachable!()
}
DirTraversalResult::Stopped => false,
}
};
debug!("check_files_name - finished checking for same names");
res
}
fn calculate_name_stats(&mut self) {
if self.use_reference_folders {
if self.common_data.use_reference_folders {
for (_fe, vector) in self.files_with_identical_names_referenced.values() {
self.information.number_of_duplicated_files_by_name += vector.len();
self.information.number_of_groups_by_name += 1;
@ -420,6 +338,7 @@ impl DuplicateFinder {
}
fn check_files_size_name(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files_size_name - starting checking for same size and name");
let group_by_func = if self.case_sensitive_name_comparison {
|fe: &FileEntry| (fe.size, fe.path.file_name().unwrap().to_string_lossy().to_string())
} else {
@ -427,23 +346,24 @@ impl DuplicateFinder {
};
let result = DirTraversalBuilder::new()
.root_dirs(self.directories.included_directories.clone())
.root_dirs(self.common_data.directories.included_directories.clone())
.group_by(group_by_func)
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.checking_method(CheckingMethod::Name)
.directories(self.directories.clone())
.allowed_extensions(self.allowed_extensions.clone())
.excluded_items(self.excluded_items.clone())
.recursive_search(self.recursive_search)
.minimal_file_size(self.minimal_file_size)
.maximal_file_size(self.maximal_file_size)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.minimal_file_size(self.common_data.minimal_file_size)
.maximal_file_size(self.common_data.maximal_file_size)
.build()
.run();
match result {
debug!("check_files_size_name - after finding file sizes");
let res = match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
self.files_with_identical_size_names = grouped_file_entries;
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
// Create new BTreeMap without single size entries(files have not duplicates)
let mut new_map: BTreeMap<(u64, String), Vec<FileEntry>> = Default::default();
@ -456,12 +376,13 @@ impl DuplicateFinder {
self.files_with_identical_size_names = new_map;
// Reference - only use in size, because later hash will be counted differently
if self.use_reference_folders {
if self.common_data.use_reference_folders {
let vec = mem::take(&mut self.files_with_identical_size_names)
.into_iter()
.filter_map(|(_size, vec_file_entry)| {
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) = vec_file_entry
.into_iter()
.partition(|e| self.common_data.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -483,11 +404,13 @@ impl DuplicateFinder {
unreachable!()
}
DirTraversalResult::Stopped => false,
}
};
debug!("check_files_size_name - finished checking for same size and name");
res
}
fn calculate_size_name_stats(&mut self) {
if self.use_reference_folders {
if self.common_data.use_reference_folders {
for ((size, _name), (_fe, vector)) in &self.files_with_identical_size_names_referenced {
self.information.number_of_duplicated_files_by_size_name += vector.len();
self.information.number_of_groups_by_size_name += 1;
@ -505,30 +428,32 @@ impl DuplicateFinder {
/// Read file length and puts it to different boxes(each for different lengths)
/// If in box is only 1 result, then it is removed
fn check_files_size(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_file_size - start");
let max_stage = match self.check_method {
CheckingMethod::Size => 0,
CheckingMethod::Hash => 2,
_ => panic!(),
};
let result = DirTraversalBuilder::new()
.root_dirs(self.directories.included_directories.clone())
.root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|fe| fe.size)
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.checking_method(self.check_method)
.max_stage(max_stage)
.directories(self.directories.clone())
.allowed_extensions(self.allowed_extensions.clone())
.excluded_items(self.excluded_items.clone())
.recursive_search(self.recursive_search)
.minimal_file_size(self.minimal_file_size)
.maximal_file_size(self.maximal_file_size)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.minimal_file_size(self.common_data.minimal_file_size)
.maximal_file_size(self.common_data.maximal_file_size)
.build()
.run();
match result {
debug!("check_file_size - after finding file sizes");
let res = match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
self.files_with_identical_size = grouped_file_entries;
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
// Create new BTreeMap without single size entries(files have not duplicates)
let old_map: BTreeMap<u64, Vec<FileEntry>> = mem::take(&mut self.files_with_identical_size);
@ -554,11 +479,13 @@ impl DuplicateFinder {
unreachable!()
}
DirTraversalResult::Stopped => false,
}
};
debug!("check_file_size - after calculating size stats/duplicates");
res
}
fn calculate_size_stats(&mut self) {
if self.use_reference_folders {
if self.common_data.use_reference_folders {
for (size, (_fe, vector)) in &self.files_with_identical_size_referenced {
self.information.number_of_duplicated_files_by_size += vector.len();
self.information.number_of_groups_by_size += 1;
@ -576,12 +503,13 @@ impl DuplicateFinder {
/// This step check for references, only when checking for size.
/// This is needed, because later reference folders looks for hashes, not size
fn filter_reference_folders_by_size(&mut self) {
if self.use_reference_folders && self.check_method == CheckingMethod::Size {
if self.common_data.use_reference_folders && self.check_method == CheckingMethod::Size {
let vec = mem::take(&mut self.files_with_identical_size)
.into_iter()
.filter_map(|(_size, vec_file_entry)| {
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) = vec_file_entry
.into_iter()
.partition(|e| self.common_data.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -606,7 +534,8 @@ impl DuplicateFinder {
let mut non_cached_files_to_check: BTreeMap<u64, Vec<FileEntry>> = Default::default();
if self.use_prehash_cache {
loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache, &self.hash_type, true) {
debug!("prehash_load_cache_at_start - using prehash cache start");
loaded_hash_map = match load_hashes_from_file(&mut self.common_data.text_messages, self.common_data.delete_outdated_cache, &self.hash_type, true) {
Some(t) => t,
None => Default::default(),
};
@ -624,24 +553,27 @@ impl DuplicateFinder {
let name = file_entry.path.to_string_lossy().to_string();
if !loaded_hash_map2.contains_key(&name) {
// If loaded data doesn't contains current image info
non_cached_files_to_check.entry(file_entry.size).or_insert_with(Vec::new).push(file_entry.clone());
non_cached_files_to_check.entry(file_entry.size).or_default().push(file_entry.clone());
} else if file_entry.size != loaded_hash_map2.get(&name).unwrap().size || file_entry.modified_date != loaded_hash_map2.get(&name).unwrap().modified_date {
// When size or modification date of image changed, then it is clear that is different image
non_cached_files_to_check.entry(file_entry.size).or_insert_with(Vec::new).push(file_entry.clone());
non_cached_files_to_check.entry(file_entry.size).or_default().push(file_entry.clone());
} else {
// Checking may be omitted when already there is entry with same size and modification date
records_already_cached.entry(file_entry.size).or_insert_with(Vec::new).push(file_entry.clone());
records_already_cached.entry(file_entry.size).or_default().push(file_entry.clone());
}
}
}
} else {
debug!("prehash_load_cache_at_start - not using prehash cache start");
loaded_hash_map = Default::default();
mem::swap(&mut self.files_with_identical_size, &mut non_cached_files_to_check);
}
debug!("prehash_load_cache_at_start - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
fn prehash_save_cache_at_exit(&mut self, loaded_hash_map: BTreeMap<u64, Vec<FileEntry>>, pre_hash_results: &Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)>) {
debug!("prehash_save_cache_at_exit - start - using prehash cache {}", self.use_prehash_cache);
if self.use_prehash_cache {
// All results = records already cached + computed results
let mut save_cache_to_hashmap: BTreeMap<String, FileEntry> = Default::default();
@ -664,7 +596,14 @@ impl DuplicateFinder {
}
}
save_hashes_to_file(&save_cache_to_hashmap, &mut self.text_messages, &self.hash_type, true, self.minimal_prehash_cache_file_size);
save_hashes_to_file(
&save_cache_to_hashmap,
&mut self.common_data.text_messages,
&self.hash_type,
true,
self.minimal_prehash_cache_file_size,
);
debug!("prehash_save_cache_at_exit - saving prehash cache end");
}
}
@ -674,6 +613,7 @@ impl DuplicateFinder {
progress_sender: Option<&UnboundedSender<ProgressData>>,
pre_checked_map: &mut BTreeMap<u64, Vec<FileEntry>>,
) -> Option<()> {
debug!("prehashing - start");
let check_type = self.hash_type;
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = prepare_thread_handler_common(
progress_sender,
@ -681,7 +621,7 @@ impl DuplicateFinder {
2,
self.files_with_identical_size.values().map(Vec::len).sum(),
self.check_method,
self.tool_type,
self.common_data.tool_type,
);
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.prehash_load_cache_at_start();
@ -702,7 +642,7 @@ impl DuplicateFinder {
for file_entry in vec_file_entry {
match hash_calculation(&mut buffer, file_entry, &check_type, 0) {
Ok(hash_string) => {
hashmap_with_hash.entry(hash_string.clone()).or_insert_with(Vec::new).push(file_entry.clone());
hashmap_with_hash.entry(hash_string.clone()).or_default().push(file_entry.clone());
}
Err(s) => errors.push(s),
}
@ -721,21 +661,22 @@ impl DuplicateFinder {
// Add data from cache
for (size, vec_file_entry) in &records_already_cached {
pre_checked_map.entry(*size).or_insert_with(Vec::new).append(&mut vec_file_entry.clone());
pre_checked_map.entry(*size).or_default().append(&mut vec_file_entry.clone());
}
// Check results
for (size, hash_map, errors) in &pre_hash_results {
self.text_messages.warnings.append(&mut errors.clone());
self.common_data.text_messages.warnings.append(&mut errors.clone());
for vec_file_entry in hash_map.values() {
if vec_file_entry.len() > 1 {
pre_checked_map.entry(*size).or_insert_with(Vec::new).append(&mut vec_file_entry.clone());
pre_checked_map.entry(*size).or_default().append(&mut vec_file_entry.clone());
}
}
}
self.prehash_save_cache_at_exit(loaded_hash_map, &pre_hash_results);
debug!("prehashing - end");
Some(())
}
@ -743,12 +684,14 @@ impl DuplicateFinder {
&mut self,
mut pre_checked_map: BTreeMap<u64, Vec<FileEntry>>,
) -> (BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>) {
debug!("full_hashing_load_cache_at_start - start");
let loaded_hash_map;
let mut records_already_cached: BTreeMap<u64, Vec<FileEntry>> = Default::default();
let mut non_cached_files_to_check: BTreeMap<u64, Vec<FileEntry>> = Default::default();
if self.use_cache {
loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache, &self.hash_type, false) {
if self.common_data.use_cache {
debug!("full_hashing_load_cache_at_start - using cache");
loaded_hash_map = match load_hashes_from_file(&mut self.common_data.text_messages, self.common_data.delete_outdated_cache, &self.hash_type, false) {
Some(t) => t,
None => Default::default(),
};
@ -765,22 +708,24 @@ impl DuplicateFinder {
let mut found: bool = false;
for loaded_file_entry in loaded_vec_file_entry {
if file_entry.path == loaded_file_entry.path && file_entry.modified_date == loaded_file_entry.modified_date {
records_already_cached.entry(file_entry.size).or_insert_with(Vec::new).push(loaded_file_entry.clone());
records_already_cached.entry(file_entry.size).or_default().push(loaded_file_entry.clone());
found = true;
break;
}
}
if !found {
non_cached_files_to_check.entry(file_entry.size).or_insert_with(Vec::new).push(file_entry);
non_cached_files_to_check.entry(file_entry.size).or_default().push(file_entry);
}
}
}
}
} else {
debug!("full_hashing_load_cache_at_start - not using cache");
loaded_hash_map = Default::default();
mem::swap(&mut pre_checked_map, &mut non_cached_files_to_check);
}
debug!("full_hashing_load_cache_at_start - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
@ -790,7 +735,8 @@ impl DuplicateFinder {
full_hash_results: &mut Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)>,
loaded_hash_map: BTreeMap<u64, Vec<FileEntry>>,
) {
if !self.use_cache {
debug!("full_hashing_save_cache_at_exit - start");
if !self.common_data.use_cache {
return;
}
'main: for (size, vec_file_entry) in records_already_cached {
@ -798,7 +744,7 @@ impl DuplicateFinder {
for (full_size, full_hashmap, _errors) in &mut (*full_hash_results) {
if size == *full_size {
for file_entry in vec_file_entry {
full_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry);
full_hashmap.entry(file_entry.hash.clone()).or_default().push(file_entry);
}
continue 'main;
}
@ -806,7 +752,7 @@ impl DuplicateFinder {
// Size doesn't exists add results to files
let mut temp_hashmap: BTreeMap<String, Vec<FileEntry>> = Default::default();
for file_entry in vec_file_entry {
temp_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry);
temp_hashmap.entry(file_entry.hash.clone()).or_default().push(file_entry);
}
full_hash_results.push((size, temp_hashmap, Vec::new()));
}
@ -825,7 +771,8 @@ impl DuplicateFinder {
}
}
}
save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type, false, self.minimal_cache_file_size);
save_hashes_to_file(&all_results, &mut self.common_data.text_messages, &self.hash_type, false, self.minimal_cache_file_size);
debug!("full_hashing_save_cache_at_exit - end");
}
fn full_hashing(
@ -834,10 +781,17 @@ impl DuplicateFinder {
progress_sender: Option<&UnboundedSender<ProgressData>>,
pre_checked_map: BTreeMap<u64, Vec<FileEntry>>,
) -> Option<()> {
debug!("full_hashing - start");
let check_type = self.hash_type;
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 2, pre_checked_map.values().map(Vec::len).sum(), self.check_method, self.tool_type);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = prepare_thread_handler_common(
progress_sender,
2,
2,
pre_checked_map.values().map(Vec::len).sum(),
self.check_method,
self.common_data.tool_type,
);
///////////////////////////////////////////////////////////////////////////// HASHING START
{
@ -860,7 +814,7 @@ impl DuplicateFinder {
match hash_calculation(&mut buffer, &file_entry, &check_type, u64::MAX) {
Ok(hash_string) => {
file_entry.hash = hash_string.clone();
hashmap_with_hash.entry(hash_string.clone()).or_insert_with(Vec::new).push(file_entry);
hashmap_with_hash.entry(hash_string.clone()).or_default().push(file_entry);
}
Err(s) => errors.push(s),
}
@ -880,28 +834,30 @@ impl DuplicateFinder {
}
for (size, hash_map, mut errors) in full_hash_results {
self.text_messages.warnings.append(&mut errors);
self.common_data.text_messages.warnings.append(&mut errors);
for (_hash, vec_file_entry) in hash_map {
if vec_file_entry.len() > 1 {
self.files_with_identical_hashes.entry(size).or_insert_with(Vec::new).push(vec_file_entry);
self.files_with_identical_hashes.entry(size).or_default().push(vec_file_entry);
}
}
}
}
debug!("full_hashing - end");
Some(())
}
fn hash_reference_folders(&mut self) {
// Reference - only use in size, because later hash will be counted differently
if self.use_reference_folders {
if self.common_data.use_reference_folders {
let vec = mem::take(&mut self.files_with_identical_hashes)
.into_iter()
.filter_map(|(_size, vec_vec_file_entry)| {
let mut all_results_with_same_size = Vec::new();
for vec_file_entry in vec_vec_file_entry {
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) = vec_file_entry
.into_iter()
.partition(|e| self.common_data.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
continue;
@ -920,7 +876,7 @@ impl DuplicateFinder {
}
}
if self.use_reference_folders {
if self.common_data.use_reference_folders {
for (size, vector_vectors) in &self.files_with_identical_hashes_referenced {
for (_fe, vector) in vector_vectors {
self.information.number_of_duplicated_files_by_hash += vector.len();
@ -972,24 +928,24 @@ impl DuplicateFinder {
match self.check_method {
CheckingMethod::Name => {
for vector in self.files_with_identical_names.values() {
let _tuple: (u64, usize, usize) = delete_files(vector, &self.delete_method, &mut self.text_messages, self.dryrun);
let _tuple: (u64, usize, usize) = delete_files(vector, &self.delete_method, &mut self.common_data.text_messages, self.dryrun);
}
}
CheckingMethod::SizeName => {
for vector in self.files_with_identical_size_names.values() {
let _tuple: (u64, usize, usize) = delete_files(vector, &self.delete_method, &mut self.text_messages, self.dryrun);
let _tuple: (u64, usize, usize) = delete_files(vector, &self.delete_method, &mut self.common_data.text_messages, self.dryrun);
}
}
CheckingMethod::Hash => {
for vector_vectors in self.files_with_identical_hashes.values() {
for vector in vector_vectors.iter() {
let _tuple: (u64, usize, usize) = delete_files(vector, &self.delete_method, &mut self.text_messages, self.dryrun);
for vector in vector_vectors {
let _tuple: (u64, usize, usize) = delete_files(vector, &self.delete_method, &mut self.common_data.text_messages, self.dryrun);
}
}
}
CheckingMethod::Size => {
for vector in self.files_with_identical_size.values() {
let _tuple: (u64, usize, usize) = delete_files(vector, &self.delete_method, &mut self.text_messages, self.dryrun);
let _tuple: (u64, usize, usize) = delete_files(vector, &self.delete_method, &mut self.common_data.text_messages, self.dryrun);
}
}
_ => panic!(),
@ -1013,11 +969,6 @@ impl DebugPrint for DuplicateFinder {
return;
}
println!("---------------DEBUG PRINT---------------");
println!("### Information's");
println!("Errors size - {}", self.text_messages.errors.len());
println!("Warnings size - {}", self.text_messages.warnings.len());
println!("Messages size - {}", self.text_messages.messages.len());
println!(
"Number of duplicated files by size(in groups) - {} ({})",
self.information.number_of_duplicated_files_by_size, self.information.number_of_groups_by_size
@ -1045,15 +996,9 @@ impl DebugPrint for DuplicateFinder {
println!("Files list size - {}", self.files_with_identical_size.len());
println!("Hashed Files list size - {}", self.files_with_identical_hashes.len());
println!("Excluded items - {:?}", self.excluded_items.items);
println!("Included directories - {:?}", self.directories.included_directories);
println!("Excluded directories - {:?}", self.directories.excluded_directories);
println!("Recursive search - {}", self.recursive_search);
#[cfg(target_family = "unix")]
println!("Skip other filesystems - {}", self.directories.exclude_other_filesystems());
println!("Minimum file size - {:?}", self.minimal_file_size);
println!("Checking Method - {:?}", self.check_method);
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -1068,7 +1013,7 @@ impl SaveResults for DuplicateFinder {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -1077,9 +1022,12 @@ impl SaveResults for DuplicateFinder {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
match self.check_method {
@ -1305,7 +1253,7 @@ fn delete_files(vector: &[FileEntry], delete_method: &DeleteMethod, text_message
if dryrun {
Ok(Some(format!("Delete {}", file.path.display())))
} else {
fs::remove_file(&file.path).map(|_| None)
fs::remove_file(&file.path).map(|()| None)
}
}
DeleteMethod::HardLink => {
@ -1313,7 +1261,7 @@ fn delete_files(vector: &[FileEntry], delete_method: &DeleteMethod, text_message
if dryrun {
Ok(Some(format!("Replace file {} with hard link to {}", file.path.display(), src.display())))
} else {
make_hard_link(src, &file.path).map(|_| None)
make_hard_link(src, &file.path).map(|()| None)
}
}
DeleteMethod::None => Ok(None),
@ -1401,7 +1349,9 @@ pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache
open_cache_folder(&get_file_hash_name(type_of_hash, is_prehash), false, false, &mut text_messages.warnings)
{
// Unwrap could fail when failed to open cache file, but json would exists
let Some(file_handler) = file_handler else { return Default::default(); };
let Some(file_handler) = file_handler else {
return Default::default();
};
let reader = BufReader::new(file_handler);
let mut hashmap_loaded_entries: BTreeMap<u64, Vec<FileEntry>> = Default::default();
@ -1460,7 +1410,7 @@ pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache
hash: uuu[3].to_string(),
symlink_info: None,
};
hashmap_loaded_entries.entry(file_entry.size).or_insert_with(Vec::new).push(file_entry);
hashmap_loaded_entries.entry(file_entry.size).or_default().push(file_entry);
}
}
@ -1535,6 +1485,15 @@ impl MyHasher for Xxh3 {
}
}
impl CommonData for DuplicateFinder {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}
#[cfg(test)]
mod tests {
use std::fs::{read_dir, File, Metadata};

165
czkawka_core/src/empty_files.rs
View file

@ -2,16 +2,13 @@ use std::fs;
use std::fs::File;
use std::io::prelude::*;
use std::io::BufWriter;
use std::path::PathBuf;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use log::{debug, info};
use crate::common_dir_traversal::{DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::*;
#[derive(Eq, PartialEq, Clone, Debug)]
@ -26,128 +23,74 @@ pub struct Info {
pub number_of_empty_files: usize,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
/// Struct with required information's to work
pub struct EmptyFiles {
#[allow(dead_code)]
tool_type: ToolType,
text_messages: Messages,
common_data: CommonToolData,
information: Info,
empty_files: Vec<FileEntry>,
directories: Directories,
allowed_extensions: Extensions,
excluded_items: ExcludedItems,
recursive_search: bool,
delete_method: DeleteMethod,
stopped_search: bool,
}
impl CommonData for EmptyFiles {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}
impl EmptyFiles {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::EmptyFiles,
text_messages: Messages::new(),
information: Info::new(),
recursive_search: true,
allowed_extensions: Extensions::new(),
directories: Directories::new(),
excluded_items: ExcludedItems::new(),
common_data: CommonToolData::new(ToolType::EmptyFiles),
information: Info::default(),
empty_files: vec![],
delete_method: DeleteMethod::None,
stopped_search: false,
}
}
/// Finding empty files, save results to internal struct variables
pub fn find_empty_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.directories.optimize_directories(self.recursive_search, &mut self.text_messages);
info!("Starting finding empty files");
let start_time = std::time::Instant::now();
self.find_empty_files_internal(stop_receiver, progress_sender);
info!("Ended finding empty files which took {:?}", start_time.elapsed());
}
fn find_empty_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
self.delete_files();
self.debug_print();
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
#[must_use]
pub const fn get_empty_files(&self) -> &Vec<FileEntry> {
&self.empty_files
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) -> bool {
self.directories.set_included_directory(included_directory, &mut self.text_messages)
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
pub fn set_allowed_extensions(&mut self, allowed_extensions: String) {
self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages);
}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
/// Check files for any with size == 0
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let result = DirTraversalBuilder::new()
.root_dirs(self.directories.included_directories.clone())
.root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ())
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.minimal_file_size(0)
.maximal_file_size(0)
.directories(self.directories.clone())
.allowed_extensions(self.allowed_extensions.clone())
.excluded_items(self.excluded_items.clone())
.recursive_search(self.recursive_search)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.build()
.run();
match result {
debug!("check_files - collected files to check");
let res = match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(empty_files) = grouped_file_entries.get(&()) {
self.empty_files = empty_files.clone();
}
self.information.number_of_empty_files = self.empty_files.len();
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
true
}
@ -155,7 +98,9 @@ impl EmptyFiles {
unreachable!()
}
DirTraversalResult::Stopped => false,
}
};
debug!("check_files - end");
res
}
/// Function to delete files, from filed Vector
@ -164,7 +109,7 @@ impl EmptyFiles {
DeleteMethod::Delete => {
for file_entry in &self.empty_files {
if fs::remove_file(file_entry.path.clone()).is_err() {
self.text_messages.warnings.push(file_entry.path.display().to_string());
self.common_data.text_messages.warnings.push(file_entry.path.display().to_string());
}
}
}
@ -191,22 +136,9 @@ impl DebugPrint for EmptyFiles {
return;
}
println!("---------------DEBUG PRINT---------------");
println!("### Information's");
println!("Errors size - {}", self.text_messages.errors.len());
println!("Warnings size - {}", self.text_messages.warnings.len());
println!("Messages size - {}", self.text_messages.messages.len());
println!("### Other");
println!("Empty list size - {}", self.empty_files.len());
println!("Excluded items - {:?}", self.excluded_items.items);
println!("Included directories - {:?}", self.directories.included_directories);
println!("Excluded directories - {:?}", self.directories.excluded_directories);
println!("Recursive search - {}", self.recursive_search);
#[cfg(target_family = "unix")]
println!("Skip other filesystems - {}", self.directories.exclude_other_filesystems());
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -221,7 +153,7 @@ impl SaveResults for EmptyFiles {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -230,9 +162,12 @@ impl SaveResults for EmptyFiles {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -259,3 +194,17 @@ impl PrintResults for EmptyFiles {
}
}
}
impl EmptyFiles {
pub const fn get_empty_files(&self) -> &Vec<FileEntry> {
&self.empty_files
}
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
}

113
czkawka_core/src/empty_folder.rs
View file

@ -6,24 +6,18 @@ use std::path::PathBuf;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use log::{debug, info};
use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, FolderEmptiness, FolderEntry, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::{DebugPrint, PrintResults, SaveResults};
/// Struct to store most basics info about all folder
pub struct EmptyFolder {
#[allow(dead_code)]
tool_type: ToolType,
common_data: CommonToolData,
information: Info,
delete_folders: bool,
text_messages: Messages,
excluded_items: ExcludedItems,
empty_folder_list: BTreeMap<PathBuf, FolderEntry>, // Path, FolderEntry
directories: Directories,
stopped_search: bool,
}
/// Info struck with helpful information's about results
@ -32,68 +26,38 @@ pub struct Info {
pub number_of_empty_folders: usize,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
/// Method implementation for `EmptyFolder`
impl EmptyFolder {
/// New function providing basics values
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::EmptyFolders,
common_data: CommonToolData::new(ToolType::EmptyFolders),
information: Default::default(),
delete_folders: false,
text_messages: Messages::new(),
excluded_items: Default::default(),
empty_folder_list: Default::default(),
directories: Directories::new(),
stopped_search: false,
}
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
#[must_use]
pub const fn get_empty_folder_list(&self) -> &BTreeMap<PathBuf, FolderEntry> {
&self.empty_folder_list
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
/// Public function used by CLI to search for empty folders
pub fn find_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.directories.optimize_directories(true, &mut self.text_messages);
info!("Starting finding empty folders");
let start_time = std::time::Instant::now();
self.find_empty_folders_internal(stop_receiver, progress_sender);
info!("Ended finding empty folders which took {:?}", start_time.elapsed());
}
fn find_empty_folders_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
if !self.check_for_empty_folders(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
self.optimize_folders();
@ -131,18 +95,20 @@ impl EmptyFolder {
/// Function to check if folder are empty.
/// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty
fn check_for_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_empty_folders - start");
let result = DirTraversalBuilder::new()
.root_dirs(self.directories.included_directories.clone())
.root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ())
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.directories(self.directories.clone())
.excluded_items(self.excluded_items.clone())
.directories(self.common_data.directories.clone())
.excluded_items(self.common_data.excluded_items.clone())
.collect(Collect::EmptyFolders)
.max_stage(0)
.build()
.run();
match result {
debug!("check_for_empty_folders - collected folders to check");
let res = match result {
DirTraversalResult::SuccessFiles { .. } => {
unreachable!()
}
@ -155,12 +121,14 @@ impl EmptyFolder {
}
}
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
true
}
DirTraversalResult::Stopped => false,
}
};
debug!("check_for_empty_folders - end");
res
}
/// Deletes earlier found empty folders
@ -168,16 +136,15 @@ impl EmptyFolder {
// Folders may be deleted or require too big privileges
for name in self.empty_folder_list.keys() {
match fs::remove_dir_all(name) {
Ok(_) => (),
Err(e) => self.text_messages.warnings.push(format!("Failed to remove folder {}, reason {}", name.display(), e)),
Ok(()) => (),
Err(e) => self
.common_data
.text_messages
.warnings
.push(format!("Failed to remove folder {}, reason {}", name.display(), e)),
};
}
}
/// Set included dir which needs to be relative, exists etc.
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) {
self.directories.set_included_directory(included_directory, &mut self.text_messages);
}
}
impl Default for EmptyFolder {
@ -197,7 +164,7 @@ impl DebugPrint for EmptyFolder {
println!("---------------DEBUG PRINT---------------");
println!("Number of empty folders - {}", self.information.number_of_empty_folders);
println!("Included directories - {:?}", self.directories.included_directories);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -212,7 +179,7 @@ impl SaveResults for EmptyFolder {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -221,9 +188,12 @@ impl SaveResults for EmptyFolder {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?}",
self.directories.included_directories, self.directories.excluded_directories
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -255,3 +225,12 @@ impl PrintResults for EmptyFolder {
}
}
}
impl CommonData for EmptyFolder {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}

138
czkawka_core/src/invalid_symlinks.rs
View file

@ -2,16 +2,13 @@ use std::fs;
use std::fs::File;
use std::io::prelude::*;
use std::io::BufWriter;
use std::path::PathBuf;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use log::{debug, info};
use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, ErrorType, FileEntry, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::*;
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
@ -26,71 +23,45 @@ pub struct Info {
pub number_of_invalid_symlinks: usize,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
/// Struct with required information's to work
pub struct InvalidSymlinks {
#[allow(dead_code)]
tool_type: ToolType,
text_messages: Messages,
common_data: CommonToolData,
information: Info,
invalid_symlinks: Vec<FileEntry>,
directories: Directories,
allowed_extensions: Extensions,
excluded_items: ExcludedItems,
recursive_search: bool,
delete_method: DeleteMethod,
stopped_search: bool,
}
impl InvalidSymlinks {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::InvalidSymlinks,
text_messages: Messages::new(),
information: Info::new(),
recursive_search: true,
allowed_extensions: Extensions::new(),
directories: Directories::new(),
excluded_items: ExcludedItems::new(),
common_data: CommonToolData::new(ToolType::InvalidSymlinks),
information: Info::default(),
invalid_symlinks: vec![],
delete_method: DeleteMethod::None,
stopped_search: false,
}
}
pub fn find_invalid_links(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.directories.optimize_directories(self.recursive_search, &mut self.text_messages);
info!("Starting finding invalid symlinks");
let start_time = std::time::Instant::now();
self.find_invalid_links_internal(stop_receiver, progress_sender);
info!("Ended finding invalid symlinks which took {:?}", start_time.elapsed());
}
fn find_invalid_links_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
self.delete_files();
self.debug_print();
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
#[must_use]
pub const fn get_invalid_symlinks(&self) -> &Vec<FileEntry> {
&self.invalid_symlinks
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
@ -99,58 +70,36 @@ impl InvalidSymlinks {
self.delete_method = delete_method;
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) -> bool {
self.directories.set_included_directory(included_directory, &mut self.text_messages)
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
pub fn set_allowed_extensions(&mut self, allowed_extensions: String) {
self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages);
}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
/// Check files for any with size == 0
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let result = DirTraversalBuilder::new()
.root_dirs(self.directories.included_directories.clone())
.root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ())
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.collect(Collect::InvalidSymlinks)
.directories(self.directories.clone())
.allowed_extensions(self.allowed_extensions.clone())
.excluded_items(self.excluded_items.clone())
.recursive_search(self.recursive_search)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.build()
.run();
match result {
debug!("check_files - collected files");
let res = match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
if let Some(((), invalid_symlinks)) = grouped_file_entries.into_iter().next() {
self.invalid_symlinks = invalid_symlinks;
}
self.information.number_of_invalid_symlinks = self.invalid_symlinks.len();
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
true
}
DirTraversalResult::SuccessFolders { .. } => unreachable!(),
DirTraversalResult::Stopped => false,
}
};
debug!("check_files - end");
res
}
/// Function to delete files, from filed Vector
@ -159,7 +108,7 @@ impl InvalidSymlinks {
DeleteMethod::Delete => {
for file_entry in &self.invalid_symlinks {
if fs::remove_file(file_entry.path.clone()).is_err() {
self.text_messages.warnings.push(file_entry.path.display().to_string());
self.common_data.text_messages.warnings.push(file_entry.path.display().to_string());
}
}
}
@ -186,22 +135,9 @@ impl DebugPrint for InvalidSymlinks {
return;
}
println!("---------------DEBUG PRINT---------------");
println!("### Information's");
println!("Errors size - {}", self.text_messages.errors.len());
println!("Warnings size - {}", self.text_messages.warnings.len());
println!("Messages size - {}", self.text_messages.messages.len());
println!("### Other");
println!("Invalid symlinks list size - {}", self.invalid_symlinks.len());
println!("Excluded items - {:?}", self.excluded_items.items);
println!("Included directories - {:?}", self.directories.included_directories);
println!("Excluded directories - {:?}", self.directories.excluded_directories);
println!("Recursive search - {}", self.recursive_search);
#[cfg(target_family = "unix")]
println!("Skip other filesystems - {}", self.directories.exclude_other_filesystems());
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -216,7 +152,7 @@ impl SaveResults for InvalidSymlinks {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -225,9 +161,12 @@ impl SaveResults for InvalidSymlinks {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -271,3 +210,12 @@ impl PrintResults for InvalidSymlinks {
}
}
}
impl CommonData for InvalidSymlinks {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}

1
czkawka_core/src/lib.rs
View file

@ -24,6 +24,7 @@ pub mod common_directory;
pub mod common_extensions;
pub mod common_items;
pub mod common_messages;
pub mod common_tool;
pub mod common_traits;
pub mod localizer_core;

5
czkawka_core/src/localizer_core.rs
View file

@ -29,12 +29,11 @@ macro_rules! flc {
}
// Get the `Localizer` to be used for localizing this library.
#[must_use]
pub fn localizer_core() -> Box<dyn Localizer> {
Box::from(DefaultLocalizer::new(&*LANGUAGE_LOADER_CORE, &Localizations))
}
#[must_use]
pub fn generate_translation_hashmap(vec: Vec<(&'static str, String)>) -> HashMap<&'static str, String> {
let mut hashmap: HashMap<&'static str, String> = Default::default();
for (key, value) in vec {
@ -43,12 +42,10 @@ pub fn generate_translation_hashmap(vec: Vec<(&'static str, String)>) -> HashMap
hashmap
}
#[must_use]
pub fn fnc_get_similarity_very_high() -> String {
flc!("core_similarity_very_high")
}
#[must_use]
pub fn fnc_get_similarity_minimal() -> String {
flc!("core_similarity_minimal")
}

383
czkawka_core/src/same_music.rs
View file

@ -12,6 +12,7 @@ use anyhow::Context;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use lofty::{read_from, AudioFile, ItemKey, TaggedFileExt};
use log::{debug, info};
use rayon::prelude::*;
use rusty_chromaprint::{match_fingerprints, Configuration, Fingerprinter};
use serde::{Deserialize, Serialize};
@ -26,10 +27,8 @@ use crate::common::{
create_crash_message, filter_reference_folders_generic, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS,
};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::*;
#[derive(Eq, PartialEq, Clone, Debug, Copy)]
@ -99,36 +98,17 @@ pub struct Info {
pub number_of_groups: u64,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
/// Struct with required information's to work
pub struct SameMusic {
tool_type: ToolType,
text_messages: Messages,
common_data: CommonToolData,
information: Info,
music_to_check: HashMap<String, MusicEntry>,
music_entries: Vec<MusicEntry>,
duplicated_music_entries: Vec<Vec<MusicEntry>>,
duplicated_music_entries_referenced: Vec<(MusicEntry, Vec<MusicEntry>)>,
directories: Directories,
allowed_extensions: Extensions,
excluded_items: ExcludedItems,
minimal_file_size: u64,
maximal_file_size: u64,
recursive_search: bool,
delete_method: DeleteMethod,
music_similarity: MusicSimilarity,
stopped_search: bool,
approximate_comparison: bool,
use_cache: bool,
delete_outdated_cache: bool, // TODO add this to GUI
use_reference_folders: bool,
save_also_as_json: bool,
check_type: CheckingMethod,
hash_preset_config: Configuration,
minimum_segment_duration: f32,
@ -136,30 +116,17 @@ pub struct SameMusic {
}
impl SameMusic {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::SameMusic,
text_messages: Messages::new(),
information: Info::new(),
recursive_search: true,
directories: Directories::new(),
allowed_extensions: Extensions::new(),
excluded_items: ExcludedItems::new(),
common_data: CommonToolData::new(ToolType::SameMusic),
information: Info::default(),
music_entries: Vec::with_capacity(2048),
delete_method: DeleteMethod::None,
music_similarity: MusicSimilarity::NONE,
stopped_search: false,
minimal_file_size: 8192,
maximal_file_size: u64::MAX,
duplicated_music_entries: vec![],
music_to_check: Default::default(),
approximate_comparison: true,
use_cache: false,
delete_outdated_cache: true,
use_reference_folders: false,
duplicated_music_entries_referenced: vec![],
save_also_as_json: false,
check_type: CheckingMethod::AudioContent,
hash_preset_config: Configuration::preset_test1(), // TODO allow to change this
minimum_segment_duration: 10.0,
@ -168,34 +135,41 @@ impl SameMusic {
}
pub fn find_same_music(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.directories.optimize_directories(self.recursive_search, &mut self.text_messages);
self.use_reference_folders = !self.directories.reference_directories.is_empty();
info!("Starting finding same music files");
let start_time = std::time::Instant::now();
self.find_same_music_internal(stop_receiver, progress_sender);
info!("Ended finding same music which took {:?}", start_time.elapsed());
}
fn find_same_music_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
self.common_data.use_reference_folders = !self.common_data.directories.reference_directories.is_empty();
if !self.check_files(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
match self.check_type {
CheckingMethod::AudioTags => {
if !self.read_tags(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
if !self.check_for_duplicate_tags(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
}
CheckingMethod::AudioContent => {
if !self.calculate_fingerprint(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
if !self.check_for_duplicate_fingerprints(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
if !self.read_tags_to_files_similar_by_content(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
}
@ -205,146 +179,27 @@ impl SameMusic {
self.debug_print();
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
#[must_use]
pub const fn get_duplicated_music_entries(&self) -> &Vec<Vec<MusicEntry>> {
&self.duplicated_music_entries
}
#[must_use]
pub const fn get_music_similarity(&self) -> &MusicSimilarity {
&self.music_similarity
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
pub fn set_save_also_as_json(&mut self, save_also_as_json: bool) {
self.save_also_as_json = save_also_as_json;
}
pub fn set_use_cache(&mut self, use_cache: bool) {
self.use_cache = use_cache;
}
pub fn set_approximate_comparison(&mut self, approximate_comparison: bool) {
self.approximate_comparison = approximate_comparison;
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
/// Set included dir which needs to be relative, exists etc.
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) {
self.directories.set_included_directory(included_directory, &mut self.text_messages);
}
pub fn set_maximum_difference(&mut self, maximum_difference: f64) {
self.maximum_difference = maximum_difference;
}
pub fn set_minimum_segment_duration(&mut self, minimum_segment_duration: f32) {
self.minimum_segment_duration = minimum_segment_duration;
}
pub fn set_reference_directory(&mut self, reference_directory: Vec<PathBuf>) {
self.directories.set_reference_directory(reference_directory);
}
pub fn set_check_type(&mut self, check_type: CheckingMethod) {
assert!([CheckingMethod::AudioTags, CheckingMethod::AudioContent].contains(&check_type));
self.check_type = check_type;
}
#[must_use]
pub fn get_check_type(&self) -> CheckingMethod {
self.check_type
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
pub fn set_allowed_extensions(&mut self, allowed_extensions: String) {
self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages);
}
pub fn set_music_similarity(&mut self, music_similarity: MusicSimilarity) {
self.music_similarity = music_similarity;
}
pub fn set_maximal_file_size(&mut self, maximal_file_size: u64) {
self.maximal_file_size = match maximal_file_size {
0 => 1,
t => t,
};
}
#[must_use]
pub fn get_similar_music_referenced(&self) -> &Vec<(MusicEntry, Vec<MusicEntry>)> {
&self.duplicated_music_entries_referenced
}
#[must_use]
pub fn get_number_of_base_duplicated_files(&self) -> usize {
if self.use_reference_folders {
self.duplicated_music_entries_referenced.len()
} else {
self.duplicated_music_entries.len()
}
}
#[must_use]
pub fn get_use_reference(&self) -> bool {
self.use_reference_folders
}
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
if !self.allowed_extensions.using_custom_extensions() {
self.allowed_extensions.extend_allowed_extensions(AUDIO_FILES_EXTENSIONS);
if !self.common_data.allowed_extensions.using_custom_extensions() {
self.common_data.allowed_extensions.extend_allowed_extensions(AUDIO_FILES_EXTENSIONS);
} else {
self.allowed_extensions.validate_allowed_extensions(AUDIO_FILES_EXTENSIONS);
if !self.allowed_extensions.using_custom_extensions() {
self.common_data.allowed_extensions.validate_allowed_extensions(AUDIO_FILES_EXTENSIONS);
if !self.common_data.allowed_extensions.using_custom_extensions() {
return true;
}
}
let result = DirTraversalBuilder::new()
.root_dirs(self.directories.included_directories.clone())
.root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ())
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.minimal_file_size(self.minimal_file_size)
.maximal_file_size(self.maximal_file_size)
.directories(self.directories.clone())
.allowed_extensions(self.allowed_extensions.clone())
.excluded_items(self.excluded_items.clone())
.recursive_search(self.recursive_search)
.minimal_file_size(self.common_data.minimal_file_size)
.maximal_file_size(self.common_data.maximal_file_size)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.max_stage(2)
.build()
.run();
@ -355,7 +210,7 @@ impl SameMusic {
self.music_to_check.insert(fe.path.to_string_lossy().to_string(), fe.to_music_entry());
}
}
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
true
}
@ -367,13 +222,14 @@ impl SameMusic {
}
fn load_cache(&mut self, checking_tags: bool) -> (HashMap<String, MusicEntry>, HashMap<String, MusicEntry>, HashMap<String, MusicEntry>) {
debug!("load_cache - start, using cache {}", self.common_data.use_cache);
let loaded_hash_map;
let mut records_already_cached: HashMap<String, MusicEntry> = Default::default();
let mut non_cached_files_to_check: HashMap<String, MusicEntry> = Default::default();
if self.use_cache {
loaded_hash_map = match load_cache_from_file(&mut self.text_messages, self.delete_outdated_cache, checking_tags) {
if self.common_data.use_cache {
loaded_hash_map = match load_cache_from_file(&mut self.common_data.text_messages, self.common_data.delete_outdated_cache, checking_tags) {
Some(t) => t,
None => Default::default(),
};
@ -397,11 +253,13 @@ impl SameMusic {
loaded_hash_map = Default::default();
mem::swap(&mut self.music_to_check, &mut non_cached_files_to_check);
}
debug!("load_cache - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
fn save_cache(&mut self, vec_file_entry: Vec<MusicEntry>, loaded_hash_map: HashMap<String, MusicEntry>, checking_tags: bool) {
if !self.use_cache {
debug!("save_cache - start, using cache {}", self.common_data.use_cache);
if !self.common_data.use_cache {
return;
}
// Must save all results to file, old loaded from file with all currently counted results
@ -410,14 +268,16 @@ impl SameMusic {
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
save_cache_to_file(&all_results, &mut self.text_messages, self.save_also_as_json, checking_tags);
save_cache_to_file(&all_results, &mut self.common_data.text_messages, self.common_data.save_also_as_json, checking_tags);
debug!("save_cache - end");
}
fn calculate_fingerprint(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("calculate_fingerprint - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache(false);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 3, non_cached_files_to_check.len(), self.check_type, self.tool_type);
prepare_thread_handler_common(progress_sender, 1, 3, non_cached_files_to_check.len(), self.check_type, self.common_data.tool_type);
let configuration = &self.hash_preset_config;
// Clean for duplicate files
@ -430,8 +290,8 @@ impl SameMusic {
return None;
}
let Ok(fingerprint) = calc_fingerprint_helper(path, configuration) else {
return Some(None);
let Ok(fingerprint) = calc_fingerprint_helper(path, configuration) else {
return Some(None);
};
music_entry.fingerprint = fingerprint;
@ -455,16 +315,18 @@ impl SameMusic {
if check_was_stopped.load(Ordering::Relaxed) {
return false;
}
debug!("calculate_fingerprint - end");
true
}
fn read_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("read_tags - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache(true);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 2, non_cached_files_to_check.len(), self.check_type, self.tool_type);
prepare_thread_handler_common(progress_sender, 1, 2, non_cached_files_to_check.len(), self.check_type, self.common_data.tool_type);
debug!("read_tags - starting reading tags");
// Clean for duplicate files
let mut vec_file_entry = non_cached_files_to_check
.into_par_iter()
@ -484,6 +346,7 @@ impl SameMusic {
.filter(Option::is_some)
.map(Option::unwrap)
.collect::<Vec<_>>();
debug!("read_tags - ended reading tags");
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
@ -499,12 +362,15 @@ impl SameMusic {
return false;
}
debug!("read_tags - end");
true
}
fn check_for_duplicate_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_duplicate_tags - start");
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 2, self.music_to_check.len(), self.check_type, self.tool_type);
prepare_thread_handler_common(progress_sender, 2, 2, self.music_to_check.len(), self.check_type, self.common_data.tool_type);
let mut old_duplicates: Vec<Vec<MusicEntry>> = vec![self.music_entries.clone()];
let mut new_duplicates: Vec<Vec<MusicEntry>> = Vec::new();
@ -561,7 +427,7 @@ impl SameMusic {
if file_entry.bitrate != 0 {
let thing = file_entry.bitrate.to_string();
if !thing.is_empty() {
hash_map.entry(thing.clone()).or_insert_with(Vec::new).push(file_entry);
hash_map.entry(thing.clone()).or_default().push(file_entry);
}
}
}
@ -579,11 +445,11 @@ impl SameMusic {
self.duplicated_music_entries = old_duplicates;
if self.use_reference_folders {
self.duplicated_music_entries_referenced = filter_reference_folders_generic(mem::take(&mut self.duplicated_music_entries), &self.directories);
if self.common_data.use_reference_folders {
self.duplicated_music_entries_referenced = filter_reference_folders_generic(mem::take(&mut self.duplicated_music_entries), &self.common_data.directories);
}
if self.use_reference_folders {
if self.common_data.use_reference_folders {
for (_fe, vector) in &self.duplicated_music_entries_referenced {
self.information.number_of_duplicates += vector.len();
self.information.number_of_groups += 1;
@ -598,12 +464,15 @@ impl SameMusic {
// Clear unused data
self.music_entries.clear();
debug!("check_for_duplicate_tags - end");
true
}
fn read_tags_to_files_similar_by_content(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("read_tags_to_files_similar_by_content - start");
let groups_to_check = max(self.duplicated_music_entries.len(), self.duplicated_music_entries_referenced.len());
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 3, 3, groups_to_check, self.check_type, self.tool_type);
prepare_thread_handler_common(progress_sender, 3, 3, groups_to_check, self.check_type, self.common_data.tool_type);
// TODO is ther a way to just run iterator and not collect any info?
if !self.duplicated_music_entries.is_empty() {
@ -648,6 +517,7 @@ impl SameMusic {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug!("read_tags_to_files_similar_by_content - end");
!check_was_stopped.load(Ordering::Relaxed)
}
@ -655,10 +525,10 @@ impl SameMusic {
let base_files: Vec<MusicEntry>;
let files_to_compare: Vec<MusicEntry>;
if self.use_reference_folders {
if self.common_data.use_reference_folders {
(base_files, files_to_compare) = mem::take(&mut self.music_entries)
.into_iter()
.partition(|f| self.directories.is_in_referenced_directory(f.get_path()));
.partition(|f| self.common_data.directories.is_in_referenced_directory(f.get_path()));
} else {
base_files = self.music_entries.clone();
files_to_compare = mem::take(&mut self.music_entries);
@ -674,6 +544,7 @@ impl SameMusic {
base_files: Vec<MusicEntry>,
files_to_compare: &[MusicEntry],
) -> Option<Vec<Vec<MusicEntry>>> {
debug!("compare_fingerprints - start");
let mut used_paths: HashSet<String> = Default::default();
let configuration = &self.hash_preset_config;
@ -722,13 +593,15 @@ impl SameMusic {
duplicated_music_entries.push(music_entries);
}
}
debug!("compare_fingerprints - end");
Some(duplicated_music_entries)
}
fn check_for_duplicate_fingerprints(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_duplicate_fingerprints - start");
let (base_files, files_to_compare) = self.split_fingerprints_to_check();
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 3, base_files.len(), self.check_type, self.tool_type);
prepare_thread_handler_common(progress_sender, 2, 3, base_files.len(), self.check_type, self.common_data.tool_type);
let Some(duplicated_music_entries) = self.compare_fingerprints(stop_receiver, &atomic_counter, base_files, &files_to_compare) else {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
@ -739,11 +612,11 @@ impl SameMusic {
self.duplicated_music_entries = duplicated_music_entries;
if self.use_reference_folders {
self.duplicated_music_entries_referenced = filter_reference_folders_generic(mem::take(&mut self.duplicated_music_entries), &self.directories);
if self.common_data.use_reference_folders {
self.duplicated_music_entries_referenced = filter_reference_folders_generic(mem::take(&mut self.duplicated_music_entries), &self.common_data.directories);
}
if self.use_reference_folders {
if self.common_data.use_reference_folders {
for (_fe, vector) in &self.duplicated_music_entries_referenced {
self.information.number_of_duplicates += vector.len();
self.information.number_of_groups += 1;
@ -758,6 +631,7 @@ impl SameMusic {
// Clear unused data
self.music_entries.clear();
debug!("check_for_duplicate_fingerprints - end");
true
}
@ -768,6 +642,7 @@ impl SameMusic {
get_item: fn(&MusicEntry) -> &str,
approximate_comparison: bool,
) -> Vec<Vec<MusicEntry>> {
debug!("check_music_item - start");
let mut new_duplicates: Vec<_> = Default::default();
let old_duplicates_len = old_duplicates.len();
for vec_file_entry in old_duplicates {
@ -778,7 +653,7 @@ impl SameMusic {
get_approximate_conversion(&mut thing);
}
if !thing.is_empty() {
hash_map.entry(thing).or_insert_with(Vec::new).push(file_entry);
hash_map.entry(thing).or_default().push(file_entry);
}
}
for (_title, vec_file_entry) in hash_map {
@ -789,16 +664,10 @@ impl SameMusic {
}
atomic_counter.fetch_add(old_duplicates_len, Ordering::Relaxed);
debug!("check_music_item - end");
new_duplicates
}
pub fn set_minimal_file_size(&mut self, minimal_file_size: u64) {
self.minimal_file_size = match minimal_file_size {
0 => 1,
t => t,
};
}
/// Function to delete files, from filed Vector
fn delete_files(&mut self) {
@ -807,7 +676,7 @@ impl SameMusic {
// DeleteMethod::Delete => {
// for file_entry in &self.music_entries {
// if fs::remove_file(file_entry.path.clone()).is_err() {
// self.text_messages.warnings.push(file_entry.path.display().to_string());
// self.common_data.text_messages.warnings.push(file_entry.path.display().to_string());
// }
// }
// }
@ -818,6 +687,64 @@ impl SameMusic {
}
}
impl SameMusic {
pub const fn get_duplicated_music_entries(&self) -> &Vec<Vec<MusicEntry>> {
&self.duplicated_music_entries
}
pub const fn get_music_similarity(&self) -> &MusicSimilarity {
&self.music_similarity
}
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
pub fn set_approximate_comparison(&mut self, approximate_comparison: bool) {
self.approximate_comparison = approximate_comparison;
}
pub fn set_maximum_difference(&mut self, maximum_difference: f64) {
self.maximum_difference = maximum_difference;
}
pub fn set_minimum_segment_duration(&mut self, minimum_segment_duration: f32) {
self.minimum_segment_duration = minimum_segment_duration;
}
pub fn set_check_type(&mut self, check_type: CheckingMethod) {
assert!([CheckingMethod::AudioTags, CheckingMethod::AudioContent].contains(&check_type));
self.check_type = check_type;
}
pub fn get_check_type(&self) -> CheckingMethod {
self.check_type
}
pub fn set_music_similarity(&mut self, music_similarity: MusicSimilarity) {
self.music_similarity = music_similarity;
}
pub fn get_similar_music_referenced(&self) -> &Vec<(MusicEntry, Vec<MusicEntry>)> {
&self.duplicated_music_entries_referenced
}
pub fn get_number_of_base_duplicated_files(&self) -> usize {
if self.common_data.use_reference_folders {
self.duplicated_music_entries_referenced.len()
} else {
self.duplicated_music_entries.len()
}
}
pub fn get_use_reference(&self) -> bool {
self.common_data.use_reference_folders
}
}
fn save_cache_to_file(hashmap: &HashMap<String, MusicEntry>, text_messages: &mut Messages, save_also_as_json: bool, checking_tags: bool) {
if let Some(((file_handler, cache_file), (file_handler_json, cache_file_json))) =
open_cache_folder(get_cache_file(checking_tags), true, save_also_as_json, &mut text_messages.warnings)
@ -924,7 +851,9 @@ fn calc_fingerprint_helper(path: impl AsRef<Path>, config: &Configuration) -> an
let mut sample_buf = None;
loop {
let Ok(packet) = format.next_packet() else { break };
let Ok(packet) = format.next_packet() else {
break;
};
if packet.track_id() != track_id {
continue;
@ -953,7 +882,9 @@ fn calc_fingerprint_helper(path: impl AsRef<Path>, config: &Configuration) -> an
}
fn read_single_file_tag(path: &str, music_entry: &mut MusicEntry) -> bool {
let Ok(mut file) = File::open(path) else { return false; };
let Ok(mut file) = File::open(path) else {
return false;
};
let result = panic::catch_unwind(move || {
match read_from(&mut file) {
@ -1070,24 +1001,10 @@ impl DebugPrint for SameMusic {
return;
}
println!("---------------DEBUG PRINT---------------");
println!("### Information's");
println!("Errors size - {}", self.text_messages.errors.len());
println!("Warnings size - {}", self.text_messages.warnings.len());
println!("Messages size - {}", self.text_messages.messages.len());
println!("### Other");
println!("Excluded items - {:?}", self.excluded_items.items);
println!("Minimum file size - {:?}", self.minimal_file_size);
println!("Found files music - {}", self.music_entries.len());
println!("Found duplicated files music - {}", self.duplicated_music_entries.len());
println!("Included directories - {:?}", self.directories.included_directories);
println!("Excluded directories - {:?}", self.directories.excluded_directories);
println!("Recursive search - {}", self.recursive_search);
#[cfg(target_family = "unix")]
println!("Skip other filesystems - {}", self.directories.exclude_other_filesystems());
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -1102,7 +1019,7 @@ impl SaveResults for SameMusic {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -1111,9 +1028,12 @@ impl SaveResults for SameMusic {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -1235,3 +1155,12 @@ mod tests {
assert_eq!(what, "Kekistan");
}
}
impl CommonData for SameMusic {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}

448
czkawka_core/src/similar_images.rs
View file

@ -12,6 +12,7 @@ use futures::channel::mpsc::UnboundedSender;
use humansize::{format_size, BINARY};
use image::GenericImageView;
use image_hasher::{FilterType, HashAlg, HasherConfig};
use log::{debug, info};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
@ -22,10 +23,8 @@ use crate::common::{
HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS,
};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults};
use crate::flc;
@ -47,6 +46,7 @@ pub struct FileEntry {
pub hash: ImHash,
pub similarity: u32,
}
impl ResultEntry for FileEntry {
fn get_path(&self) -> &Path {
&self.path
@ -81,31 +81,19 @@ impl bk_tree::Metric<ImHash> for Hamming {
/// Struct to store most basics info about all folder
pub struct SimilarImages {
tool_type: ToolType,
common_data: CommonToolData,
information: Info,
text_messages: Messages,
directories: Directories,
allowed_extensions: Extensions,
excluded_items: ExcludedItems,
bktree: BKTree<ImHash, Hamming>,
similar_vectors: Vec<Vec<FileEntry>>,
similar_referenced_vectors: Vec<(FileEntry, Vec<FileEntry>)>,
recursive_search: bool,
minimal_file_size: u64,
maximal_file_size: u64,
image_hashes: HashMap<ImHash, Vec<FileEntry>>,
// Hashmap with image hashes and Vector with names of files
stopped_search: bool,
similarity: u32,
images_to_check: HashMap<String, FileEntry>,
hash_size: u8,
hash_alg: HashAlg,
image_filter: FilterType,
use_cache: bool,
delete_outdated_cache: bool,
exclude_images_with_same_size: bool,
use_reference_folders: bool,
save_also_as_json: bool,
}
/// Info struck with helpful information's about results
@ -115,154 +103,47 @@ pub struct Info {
pub number_of_groups: u64,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
/// Method implementation for `EmptyFolder`
impl SimilarImages {
/// New function providing basics values
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::SimilarImages,
common_data: CommonToolData::new(ToolType::SimilarImages),
information: Default::default(),
text_messages: Messages::new(),
directories: Directories::new(),
excluded_items: Default::default(),
allowed_extensions: Extensions::new(),
bktree: BKTree::new(Hamming),
similar_vectors: vec![],
similar_referenced_vectors: Default::default(),
recursive_search: true,
minimal_file_size: 1024 * 16, // 16 KB should be enough to exclude too small images from search
maximal_file_size: u64::MAX,
image_hashes: Default::default(),
stopped_search: false,
similarity: 0,
images_to_check: Default::default(),
hash_size: 8,
hash_alg: HashAlg::Gradient,
image_filter: FilterType::Lanczos3,
use_cache: true,
delete_outdated_cache: true,
exclude_images_with_same_size: false,
use_reference_folders: false,
save_also_as_json: false,
}
}
pub fn set_hash_size(&mut self, hash_size: u8) {
self.hash_size = match hash_size {
8 | 16 | 32 | 64 => hash_size,
e => {
panic!("Invalid value of hash size {e}");
}
}
}
pub fn set_delete_outdated_cache(&mut self, delete_outdated_cache: bool) {
self.delete_outdated_cache = delete_outdated_cache;
}
pub fn set_exclude_images_with_same_size(&mut self, exclude_images_with_same_size: bool) {
self.exclude_images_with_same_size = exclude_images_with_same_size;
}
pub fn set_hash_alg(&mut self, hash_alg: HashAlg) {
self.hash_alg = hash_alg;
}
pub fn set_image_filter(&mut self, image_filter: FilterType) {
self.image_filter = image_filter;
}
pub fn set_save_also_as_json(&mut self, save_also_as_json: bool) {
self.save_also_as_json = save_also_as_json;
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_similar_images(&self) -> &Vec<Vec<FileEntry>> {
&self.similar_vectors
}
#[must_use]
pub fn get_similar_images_referenced(&self) -> &Vec<(FileEntry, Vec<FileEntry>)> {
&self.similar_referenced_vectors
}
#[must_use]
pub fn get_use_reference(&self) -> bool {
self.use_reference_folders
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_use_cache(&mut self, use_cache: bool) {
self.use_cache = use_cache;
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
pub fn set_allowed_extensions(&mut self, allowed_extensions: String) {
self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages);
}
pub fn set_minimal_file_size(&mut self, minimal_file_size: u64) {
self.minimal_file_size = match minimal_file_size {
0 => 1,
t => t,
};
}
pub fn set_maximal_file_size(&mut self, maximal_file_size: u64) {
self.maximal_file_size = match maximal_file_size {
0 => 1,
t => t,
};
}
pub fn set_similarity(&mut self, similarity: u32) {
self.similarity = similarity;
}
/// Public function used by CLI to search for empty folders
pub fn find_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.directories.optimize_directories(true, &mut self.text_messages);
self.use_reference_folders = !self.directories.reference_directories.is_empty();
info!("Starting finding similar images files");
let start_time = std::time::Instant::now();
self.find_similar_images_internal(stop_receiver, progress_sender);
info!("Ended finding similar images which took {:?}", start_time.elapsed());
}
pub fn find_similar_images_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
self.common_data.use_reference_folders = !self.common_data.directories.reference_directories.is_empty();
if !self.check_for_similar_images(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
if !self.hash_images(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
if !self.find_similar_hashes(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
// if self.delete_folders {
@ -278,28 +159,30 @@ impl SimilarImages {
/// Function to check if folder are empty.
/// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty
fn check_for_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_similar_images - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
if !self.allowed_extensions.using_custom_extensions() {
self.allowed_extensions.extend_allowed_extensions(IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS);
self.allowed_extensions.extend_allowed_extensions(RAW_IMAGE_EXTENSIONS);
if !self.common_data.allowed_extensions.using_custom_extensions() {
self.common_data.allowed_extensions.extend_allowed_extensions(IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS);
self.common_data.allowed_extensions.extend_allowed_extensions(RAW_IMAGE_EXTENSIONS);
#[cfg(feature = "heif")]
self.allowed_extensions.extend_allowed_extensions(HEIC_EXTENSIONS);
self.common_data.allowed_extensions.extend_allowed_extensions(HEIC_EXTENSIONS);
} else {
self.allowed_extensions
self.common_data
.allowed_extensions
.validate_allowed_extensions(&[IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS, HEIC_EXTENSIONS].concat());
if !self.allowed_extensions.using_custom_extensions() {
if !self.common_data.allowed_extensions.using_custom_extensions() {
return true;
}
}
// Add root folders for finding
for id in &self.directories.included_directories {
for id in &self.common_data.directories.included_directories {
folders_to_check.push(id.clone());
}
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 0, 2, 0, CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 0, 2, 0, CheckingMethod::None, self.common_data.tool_type);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -329,9 +212,9 @@ impl SimilarImages {
&mut warnings,
current_folder,
entry_data,
self.recursive_search,
&self.directories,
&self.excluded_items,
self.common_data.recursive_search,
&self.common_data.directories,
&self.common_data.excluded_items,
);
} else if metadata.is_file() {
atomic_counter.fetch_add(1, Ordering::Relaxed);
@ -348,7 +231,7 @@ impl SimilarImages {
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
for (name, fe) in fe_result {
self.images_to_check.insert(name, fe);
}
@ -357,6 +240,7 @@ impl SimilarImages {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug!("check_for_similar_images - end");
true
}
@ -365,14 +249,14 @@ impl SimilarImages {
return;
};
if !self.allowed_extensions.matches_filename(&file_name_lowercase) {
if !self.common_data.allowed_extensions.matches_filename(&file_name_lowercase) {
return;
}
// Checking files
if (self.minimal_file_size..=self.maximal_file_size).contains(&metadata.len()) {
if (self.common_data.minimal_file_size..=self.common_data.maximal_file_size).contains(&metadata.len()) {
let current_file_name = current_folder.join(entry_data.file_name());
if self.excluded_items.is_excluded(&current_file_name) {
if self.common_data.excluded_items.is_excluded(&current_file_name) {
return;
}
@ -390,13 +274,20 @@ impl SimilarImages {
}
fn hash_images_load_cache(&mut self) -> (HashMap<String, FileEntry>, HashMap<String, FileEntry>, HashMap<String, FileEntry>) {
debug!("hash_images_load_cache - start, use cache: {}", self.common_data.use_cache);
let loaded_hash_map;
let mut records_already_cached: HashMap<String, FileEntry> = Default::default();
let mut non_cached_files_to_check: HashMap<String, FileEntry> = Default::default();
if self.use_cache {
loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache, self.hash_size, self.hash_alg, self.image_filter) {
if self.common_data.use_cache {
loaded_hash_map = match load_hashes_from_file(
&mut self.common_data.text_messages,
self.common_data.delete_outdated_cache,
self.hash_size,
self.hash_alg,
self.image_filter,
) {
Some(t) => t,
None => Default::default(),
};
@ -420,6 +311,7 @@ impl SimilarImages {
loaded_hash_map = Default::default();
mem::swap(&mut self.images_to_check, &mut non_cached_files_to_check);
}
debug!("hash_images_load_cache - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
@ -431,11 +323,13 @@ impl SimilarImages {
// - Join all hashes and save it to file
fn hash_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("hash_images - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.hash_images_load_cache();
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 2, non_cached_files_to_check.len(), CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 1, 2, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type);
debug!("hash_images - start hashing images");
let mut vec_file_entry: Vec<(FileEntry, ImHash)> = non_cached_files_to_check
.into_par_iter()
.map(|(_s, file_entry)| {
@ -450,6 +344,7 @@ impl SimilarImages {
.filter(Option::is_some)
.map(Option::unwrap)
.collect::<Vec<(FileEntry, ImHash)>>();
debug!("hash_images - end hashing images");
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
@ -462,11 +357,25 @@ impl SimilarImages {
for (file_entry, buf) in &vec_file_entry {
// Only use to comparing, non broken hashes(all 0 or 255 hashes means that algorithm fails to decode them because e.g. contains a log of alpha channel)
if !(buf.is_empty() || buf.iter().all(|e| *e == 0) || buf.iter().all(|e| *e == 255)) {
self.image_hashes.entry(buf.clone()).or_insert_with(Vec::<FileEntry>::new).push(file_entry.clone());
self.image_hashes.entry(buf.clone()).or_default().push(file_entry.clone());
}
}
if self.use_cache {
self.save_to_cache(vec_file_entry, loaded_hash_map);
// Break if stop was clicked after saving to cache
if check_was_stopped.load(Ordering::Relaxed) {
return false;
}
debug!("hash_images - end");
true
}
fn save_to_cache(&mut self, vec_file_entry: Vec<(FileEntry, ImHash)>, loaded_hash_map: HashMap<String, FileEntry>) {
debug!("save_to_cache - start, using cache: {}", self.common_data.use_cache);
if self.common_data.use_cache {
// Must save all results to file, old loaded from file with all currently counted results
let mut all_results: HashMap<String, FileEntry> = loaded_hash_map;
for (file_entry, _hash) in vec_file_entry {
@ -474,21 +383,16 @@ impl SimilarImages {
}
save_hashes_to_file(
&all_results,
&mut self.text_messages,
self.save_also_as_json,
&mut self.common_data.text_messages,
self.common_data.save_also_as_json,
self.hash_size,
self.hash_alg,
self.image_filter,
);
}
// Break if stop was clicked after saving to cache
if check_was_stopped.load(Ordering::Relaxed) {
return false;
}
true
debug!("save_to_cache - end");
}
fn collect_image_file_entry(&self, mut file_entry: FileEntry) -> (FileEntry, ImHash) {
let file_name_lowercase = file_entry.path.to_string_lossy().to_lowercase();
@ -562,6 +466,7 @@ impl SimilarImages {
// Split hashes at 2 parts, base hashes and hashes to compare, 3 argument is set of hashes with multiple images
fn split_hashes(&mut self, all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>) -> (Vec<ImHash>, HashSet<ImHash>) {
debug!("split_hashes - start");
let hashes_with_multiple_images: HashSet<ImHash> = all_hashed_images
.iter()
.filter_map(|(hash, vec_file_entry)| {
@ -572,16 +477,16 @@ impl SimilarImages {
})
.collect();
let mut base_hashes = Vec::new(); // Initial hashes
if self.use_reference_folders {
if self.common_data.use_reference_folders {
let mut files_from_referenced_folders: HashMap<ImHash, Vec<FileEntry>> = HashMap::new();
let mut normal_files: HashMap<ImHash, Vec<FileEntry>> = HashMap::new();
all_hashed_images.clone().into_iter().for_each(|(hash, vec_file_entry)| {
for file_entry in vec_file_entry {
if is_in_reference_folder(&self.directories.reference_directories, &file_entry.path) {
files_from_referenced_folders.entry(hash.clone()).or_insert_with(Vec::new).push(file_entry);
if is_in_reference_folder(&self.common_data.directories.reference_directories, &file_entry.path) {
files_from_referenced_folders.entry(hash.clone()).or_default().push(file_entry);
} else {
normal_files.entry(hash.clone()).or_insert_with(Vec::new).push(file_entry);
normal_files.entry(hash.clone()).or_default().push(file_entry);
}
}
});
@ -599,6 +504,7 @@ impl SimilarImages {
}
base_hashes = all_hashed_images.keys().cloned().collect::<Vec<_>>();
}
debug!("split_hashes - end");
(base_hashes, hashes_with_multiple_images)
}
@ -610,7 +516,8 @@ impl SimilarImages {
collected_similar_images: &mut HashMap<ImHash, Vec<FileEntry>>,
hashes_similarity: HashMap<ImHash, (ImHash, u32)>,
) {
if self.use_reference_folders {
debug!("collect_hash_compare_result - start, use reference: {}", self.common_data.use_reference_folders);
if self.common_data.use_reference_folders {
// This is same step as without reference folders, but also checks if children are inside/outside reference directories, because may happen, that one file is inside reference folder and other outside
// Collecting results to vector
@ -621,7 +528,7 @@ impl SimilarImages {
.get(&parent_hash)
.unwrap()
.iter()
.filter(|e| is_in_reference_folder(&self.directories.reference_directories, &e.path))
.filter(|e| is_in_reference_folder(&self.common_data.directories.reference_directories, &e.path))
.cloned()
.collect();
collected_similar_images.insert(parent_hash.clone(), vec_fe);
@ -633,7 +540,7 @@ impl SimilarImages {
.get(&child_hash)
.unwrap()
.iter()
.filter(|e| !is_in_reference_folder(&self.directories.reference_directories, &e.path))
.filter(|e| !is_in_reference_folder(&self.common_data.directories.reference_directories, &e.path))
.cloned()
.collect();
for fe in &mut vec_fe {
@ -659,6 +566,7 @@ impl SimilarImages {
collected_similar_images.get_mut(&parent_hash).unwrap().append(&mut vec_fe);
}
}
debug!("collect_hash_compare_result - end");
}
fn compare_hashes_with_non_zero_tolerance(
@ -669,11 +577,12 @@ impl SimilarImages {
stop_receiver: Option<&Receiver<()>>,
tolerance: u32,
) -> bool {
debug!("compare_hashes_with_non_zero_tolerance - start");
// Don't use hashes with multiple images in bktree, because they will always be master of group and cannot be find by other hashes
let (base_hashes, hashes_with_multiple_images) = self.split_hashes(all_hashed_images);
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 2, 2, base_hashes.len(), CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 2, 2, base_hashes.len(), CheckingMethod::None, self.common_data.tool_type);
let mut hashes_parents: HashMap<ImHash, u32> = Default::default(); // Hashes used as parent (hash, children_number_of_hash)
let mut hashes_similarity: HashMap<ImHash, (ImHash, u32)> = Default::default(); // Hashes used as child, (parent_hash, similarity)
@ -730,9 +639,10 @@ impl SimilarImages {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug_check_for_duplicated_things(self.use_reference_folders, &hashes_parents, &hashes_similarity, all_hashed_images, "LATTER");
debug_check_for_duplicated_things(self.common_data.use_reference_folders, &hashes_parents, &hashes_similarity, all_hashed_images, "LATTER");
self.collect_hash_compare_result(hashes_parents, &hashes_with_multiple_images, all_hashed_images, collected_similar_images, hashes_similarity);
debug!("compare_hashes_with_non_zero_tolerance - end");
true
}
@ -743,6 +653,7 @@ impl SimilarImages {
hashes_similarity: &mut HashMap<ImHash, (ImHash, u32)>,
hashes_with_multiple_images: &HashSet<ImHash>,
) {
debug!("connect_results - start");
for (original_hash, vec_compared_hashes) in partial_results {
let mut number_of_added_child_items = 0;
for (similarity, compared_hash) in vec_compared_hashes {
@ -795,9 +706,11 @@ impl SimilarImages {
hashes_parents.insert((*original_hash).clone(), number_of_added_child_items);
}
}
debug!("connect_results - end");
}
fn find_similar_hashes(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("find_similar_hashes - start");
if self.image_hashes.is_empty() {
return true;
}
@ -831,7 +744,7 @@ impl SimilarImages {
self.remove_multiple_records_from_reference_folders();
if self.use_reference_folders {
if self.common_data.use_reference_folders {
for (_fe, vector) in &self.similar_referenced_vectors {
self.information.number_of_duplicates += vector.len();
self.information.number_of_groups += 1;
@ -848,10 +761,12 @@ impl SimilarImages {
self.images_to_check = Default::default();
self.bktree = BKTree::new(Hamming);
debug!("find_similar_hashes - end");
true
}
fn exclude_items_with_same_size(&mut self) {
debug!("exclude_items_with_same_size - start, exclude: {}", self.exclude_images_with_same_size);
if self.exclude_images_with_same_size {
for vec_file_entry in mem::take(&mut self.similar_vectors) {
let mut bt_sizes: BTreeSet<u64> = Default::default();
@ -867,15 +782,21 @@ impl SimilarImages {
}
}
}
debug!("exclude_items_with_same_size - end");
}
fn remove_multiple_records_from_reference_folders(&mut self) {
if self.use_reference_folders {
debug!(
"remove_multiple_records_from_reference_folders - start, use reference: {}",
self.common_data.use_reference_folders
);
if self.common_data.use_reference_folders {
self.similar_referenced_vectors = mem::take(&mut self.similar_vectors)
.into_iter()
.filter_map(|vec_file_entry| {
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) = vec_file_entry
.into_iter()
.partition(|e| self.common_data.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -885,6 +806,7 @@ impl SimilarImages {
})
.collect::<Vec<(FileEntry, Vec<FileEntry>)>>();
}
debug!("remove_multiple_records_from_reference_folders - end");
}
#[allow(dead_code)]
@ -921,23 +843,6 @@ impl SimilarImages {
}
assert!(!found, "Found Invalid entries, verify errors before"); // TODO crashes with empty result with reference folder, verify why
}
/// Set included dir which needs to be relative, exists etc.
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) {
self.directories.set_included_directory(included_directory, &mut self.text_messages);
}
pub fn set_reference_directory(&mut self, reference_directory: Vec<PathBuf>) {
self.directories.set_reference_directory(reference_directory);
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
}
}
fn is_in_reference_folder(reference_directories: &[PathBuf], path: &Path) -> bool {
@ -960,7 +865,7 @@ impl DebugPrint for SimilarImages {
}
println!("---------------DEBUG PRINT---------------");
println!("Included directories - {:?}", self.directories.included_directories);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -975,7 +880,7 @@ impl SaveResults for SimilarImages {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -984,9 +889,12 @@ impl SaveResults for SimilarImages {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -1129,7 +1037,6 @@ fn get_cache_file(hash_size: &u8, hash_alg: &HashAlg, image_filter: &FilterType)
)
}
#[must_use]
pub fn get_string_from_similarity(similarity: &u32, hash_size: u8) -> String {
let index_preset = match hash_size {
8 => 0,
@ -1158,7 +1065,6 @@ pub fn get_string_from_similarity(similarity: &u32, hash_size: u8) -> String {
}
}
#[must_use]
pub fn return_similarity_from_similarity_preset(similarity_preset: &SimilarityPreset, hash_size: u8) -> u32 {
let index_preset = match hash_size {
8 => 0,
@ -1303,13 +1209,67 @@ fn debug_check_for_duplicated_things(
assert!(!found_broken_thing);
}
impl CommonData for SimilarImages {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}
impl SimilarImages {
pub fn set_hash_size(&mut self, hash_size: u8) {
self.hash_size = match hash_size {
8 | 16 | 32 | 64 => hash_size,
e => {
panic!("Invalid value of hash size {e}");
}
}
}
pub fn set_exclude_images_with_same_size(&mut self, exclude_images_with_same_size: bool) {
self.exclude_images_with_same_size = exclude_images_with_same_size;
}
pub fn set_hash_alg(&mut self, hash_alg: HashAlg) {
self.hash_alg = hash_alg;
}
pub fn set_image_filter(&mut self, image_filter: FilterType) {
self.image_filter = image_filter;
}
pub const fn get_similar_images(&self) -> &Vec<Vec<FileEntry>> {
&self.similar_vectors
}
pub fn get_similar_images_referenced(&self) -> &Vec<(FileEntry, Vec<FileEntry>)> {
&self.similar_referenced_vectors
}
pub fn get_use_reference(&self) -> bool {
self.common_data.use_reference_folders
}
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_similarity(&mut self, similarity: u32) {
self.similarity = similarity;
}
}
#[cfg(test)]
mod tests {
use bk_tree::BKTree;
use std::collections::HashMap;
use std::path::PathBuf;
use bk_tree::BKTree;
use crate::common_directory::Directories;
use crate::common_tool::CommonToolData;
use crate::similar_images::{FileEntry, Hamming, ImHash, SimilarImages};
#[test]
@ -1375,7 +1335,10 @@ mod tests {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 2,
use_reference_folders: false,
common_data: CommonToolData {
use_reference_folders: false,
..Default::default()
},
..Default::default()
};
@ -1420,9 +1383,12 @@ mod tests {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 0,
use_reference_folders: true,
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
common_data: CommonToolData {
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
..Default::default()
},
use_reference_folders: true,
..Default::default()
},
..Default::default()
@ -1443,9 +1409,12 @@ mod tests {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 0,
use_reference_folders: true,
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
common_data: CommonToolData {
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
..Default::default()
},
use_reference_folders: true,
..Default::default()
},
..Default::default()
@ -1467,9 +1436,12 @@ mod tests {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 0,
use_reference_folders: true,
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
common_data: CommonToolData {
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
..Default::default()
},
use_reference_folders: true,
..Default::default()
},
..Default::default()
@ -1495,7 +1467,10 @@ mod tests {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 1,
use_reference_folders: false,
common_data: CommonToolData {
use_reference_folders: false,
..Default::default()
},
..Default::default()
};
@ -1517,7 +1492,10 @@ mod tests {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 4,
use_reference_folders: false,
common_data: CommonToolData {
use_reference_folders: false,
..Default::default()
},
..Default::default()
};
@ -1541,14 +1519,18 @@ mod tests {
}
}
}
#[test]
fn test_reference_similarity_only_one() {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 1,
use_reference_folders: true,
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
common_data: CommonToolData {
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
..Default::default()
},
use_reference_folders: true,
..Default::default()
},
..Default::default()
@ -1567,14 +1549,18 @@ mod tests {
assert_eq!(res[0].1[0].path, PathBuf::from("/home/kk/bcd.txt"));
}
}
#[test]
fn test_reference_too_small_similarity() {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 1,
use_reference_folders: true,
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
common_data: CommonToolData {
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
..Default::default()
},
use_reference_folders: true,
..Default::default()
},
..Default::default()
@ -1596,9 +1582,12 @@ mod tests {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 1,
use_reference_folders: true,
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
common_data: CommonToolData {
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
..Default::default()
},
use_reference_folders: true,
..Default::default()
},
..Default::default()
@ -1631,9 +1620,12 @@ mod tests {
for _ in 0..100 {
let mut similar_images = SimilarImages {
similarity: 10,
use_reference_folders: true,
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
common_data: CommonToolData {
directories: Directories {
reference_directories: vec![PathBuf::from("/home/rr/")],
..Default::default()
},
use_reference_folders: true,
..Default::default()
},
..Default::default()
@ -1685,7 +1677,7 @@ mod tests {
fn add_hashes(hashmap: &mut HashMap<ImHash, Vec<FileEntry>>, file_entries: Vec<FileEntry>) {
for fe in file_entries {
hashmap.entry(fe.hash.clone()).or_insert_with(Vec::new).push(fe);
hashmap.entry(fe.hash.clone()).or_default().push(fe);
}
}

302
czkawka_core/src/similar_videos.rs
View file

@ -9,6 +9,7 @@ use crossbeam_channel::Receiver;
use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound;
use futures::channel::mpsc::UnboundedSender;
use humansize::{format_size, BINARY};
use log::{debug, info};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
@ -16,10 +17,8 @@ use vid_dup_finder_lib::{NormalizedTolerance, VideoHash};
use crate::common::{check_folder_children, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults};
use crate::flc;
use crate::localizer_core::generate_translation_hashmap;
@ -34,6 +33,7 @@ pub struct FileEntry {
pub vhash: VideoHash,
pub error: String,
}
impl ResultEntry for FileEntry {
fn get_path(&self) -> &Path {
&self.path
@ -57,26 +57,23 @@ impl bk_tree::Metric<Vec<u8>> for Hamming {
/// Struct to store most basics info about all folder
pub struct SimilarVideos {
tool_type: ToolType,
common_data: CommonToolData,
information: Info,
text_messages: Messages,
directories: Directories,
excluded_items: ExcludedItems,
allowed_extensions: Extensions,
similar_vectors: Vec<Vec<FileEntry>>,
similar_referenced_vectors: Vec<(FileEntry, Vec<FileEntry>)>,
recursive_search: bool,
minimal_file_size: u64,
maximal_file_size: u64,
videos_hashes: BTreeMap<Vec<u8>, Vec<FileEntry>>,
stopped_search: bool,
videos_to_check: BTreeMap<String, FileEntry>,
use_cache: bool,
tolerance: i32,
delete_outdated_cache: bool,
exclude_videos_with_same_size: bool,
use_reference_folders: bool,
save_also_as_json: bool,
}
impl CommonData for SimilarVideos {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}
/// Info struck with helpful information's about results
@ -86,148 +83,49 @@ pub struct Info {
pub number_of_groups: u64,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
/// Method implementation for `EmptyFolder`
impl SimilarVideos {
/// New function providing basics values
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::SimilarVideos,
common_data: CommonToolData::new(ToolType::SimilarVideos),
information: Default::default(),
text_messages: Messages::new(),
directories: Directories::new(),
excluded_items: Default::default(),
allowed_extensions: Extensions::new(),
similar_vectors: vec![],
recursive_search: true,
minimal_file_size: 1024 * 16,
maximal_file_size: u64::MAX,
videos_hashes: Default::default(),
stopped_search: false,
videos_to_check: Default::default(),
use_cache: true,
tolerance: 10,
delete_outdated_cache: false,
exclude_videos_with_same_size: false,
use_reference_folders: false,
similar_referenced_vectors: vec![],
save_also_as_json: true,
}
}
pub fn set_exclude_videos_with_same_size(&mut self, exclude_videos_with_same_size: bool) {
self.exclude_videos_with_same_size = exclude_videos_with_same_size;
}
pub fn set_delete_outdated_cache(&mut self, delete_outdated_cache: bool) {
self.delete_outdated_cache = delete_outdated_cache;
}
pub fn set_tolerance(&mut self, tolerance: i32) {
assert!((0..=MAX_TOLERANCE).contains(&tolerance));
self.tolerance = tolerance;
}
pub fn set_save_also_as_json(&mut self, save_also_as_json: bool) {
self.save_also_as_json = save_also_as_json;
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
pub fn set_allowed_extensions(&mut self, allowed_extensions: String) {
self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages);
}
#[must_use]
pub const fn get_similar_videos(&self) -> &Vec<Vec<FileEntry>> {
&self.similar_vectors
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_use_cache(&mut self, use_cache: bool) {
self.use_cache = use_cache;
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
pub fn set_minimal_file_size(&mut self, minimal_file_size: u64) {
self.minimal_file_size = match minimal_file_size {
0 => 1,
t => t,
};
}
pub fn set_maximal_file_size(&mut self, maximal_file_size: u64) {
self.maximal_file_size = match maximal_file_size {
0 => 1,
t => t,
};
}
#[must_use]
pub fn get_similar_videos_referenced(&self) -> &Vec<(FileEntry, Vec<FileEntry>)> {
&self.similar_referenced_vectors
}
#[must_use]
pub fn get_number_of_base_duplicated_files(&self) -> usize {
if self.use_reference_folders {
self.similar_referenced_vectors.len()
} else {
self.similar_vectors.len()
}
}
#[must_use]
pub fn get_use_reference(&self) -> bool {
self.use_reference_folders
}
/// Public function used by CLI to search for empty folders
pub fn find_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding similar videos");
let start_time = std::time::Instant::now();
self.find_similar_videos_internal(stop_receiver, progress_sender);
info!("Ended finding similar videos which took {:?}", start_time.elapsed());
}
fn find_similar_videos_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
if !check_if_ffmpeg_is_installed() {
self.text_messages.errors.push(flc!("core_ffmpeg_not_found"));
self.common_data.text_messages.errors.push(flc!("core_ffmpeg_not_found"));
#[cfg(target_os = "windows")]
self.text_messages.errors.push(flc!("core_ffmpeg_not_found_windows"));
self.common_data.text_messages.errors.push(flc!("core_ffmpeg_not_found_windows"));
#[cfg(target_os = "linux")]
self.text_messages.errors.push(flc!(
self.common_data.text_messages.errors.push(flc!(
"core_ffmpeg_missing_in_snap",
generate_translation_hashmap(vec![("url", "https://github.com/snapcrafters/ffmpeg/issues/73".to_string())])
));
} else {
self.directories.optimize_directories(true, &mut self.text_messages);
self.use_reference_folders = !self.directories.reference_directories.is_empty();
self.optimize_dirs_before_start();
self.common_data.use_reference_folders = !self.common_data.directories.reference_directories.is_empty();
if !self.check_for_similar_videos(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
if !self.sort_videos(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
// if self.delete_folders {
@ -244,24 +142,25 @@ impl SimilarVideos {
/// Function to check if folder are empty.
/// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty
fn check_for_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_for_similar_videos - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
if !self.allowed_extensions.using_custom_extensions() {
self.allowed_extensions.extend_allowed_extensions(VIDEO_FILES_EXTENSIONS);
if !self.common_data.allowed_extensions.using_custom_extensions() {
self.common_data.allowed_extensions.extend_allowed_extensions(VIDEO_FILES_EXTENSIONS);
} else {
self.allowed_extensions.validate_allowed_extensions(VIDEO_FILES_EXTENSIONS);
if !self.allowed_extensions.using_custom_extensions() {
self.common_data.allowed_extensions.validate_allowed_extensions(VIDEO_FILES_EXTENSIONS);
if !self.common_data.allowed_extensions.using_custom_extensions() {
return true;
}
}
// Add root folders for finding
for id in &self.directories.included_directories {
for id in &self.common_data.directories.included_directories {
folders_to_check.push(id.clone());
}
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None, self.common_data.tool_type);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -292,9 +191,9 @@ impl SimilarVideos {
&mut warnings,
current_folder,
entry_data,
self.recursive_search,
&self.directories,
&self.excluded_items,
self.common_data.recursive_search,
&self.common_data.directories,
&self.common_data.excluded_items,
);
} else if metadata.is_file() {
atomic_counter.fetch_add(1, Ordering::Relaxed);
@ -311,7 +210,7 @@ impl SimilarVideos {
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
for (name, fe) in fe_result {
self.videos_to_check.insert(name, fe);
}
@ -320,23 +219,23 @@ impl SimilarVideos {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
debug!("check_for_similar_videos - end");
true
}
fn add_video_file_entry(&self, metadata: &Metadata, entry_data: &DirEntry, fe_result: &mut Vec<(String, FileEntry)>, warnings: &mut Vec<String>, current_folder: &Path) {
let Some(file_name_lowercase) = get_lowercase_name(entry_data,
warnings) else {
let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else {
return;
};
if !self.allowed_extensions.matches_filename(&file_name_lowercase) {
if !self.common_data.allowed_extensions.matches_filename(&file_name_lowercase) {
return;
}
// Checking files
if (self.minimal_file_size..=self.maximal_file_size).contains(&metadata.len()) {
if (self.common_data.minimal_file_size..=self.common_data.maximal_file_size).contains(&metadata.len()) {
let current_file_name = current_folder.join(entry_data.file_name());
if self.excluded_items.is_excluded(&current_file_name) {
if self.common_data.excluded_items.is_excluded(&current_file_name) {
return;
}
let current_file_name_str = current_file_name.to_string_lossy().to_string();
@ -354,12 +253,13 @@ impl SimilarVideos {
}
fn load_cache_at_start(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) {
debug!("load_cache_at_start - start, use cache: {}", self.common_data.use_cache);
let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default();
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default();
if self.use_cache {
loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache) {
if self.common_data.use_cache {
loaded_hash_map = match load_hashes_from_file(&mut self.common_data.text_messages, self.common_data.delete_outdated_cache) {
Some(t) => t,
None => Default::default(),
};
@ -383,14 +283,16 @@ impl SimilarVideos {
loaded_hash_map = Default::default();
mem::swap(&mut self.videos_to_check, &mut non_cached_files_to_check);
}
debug!("load_cache_at_start - end");
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
fn sort_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("sort_videos - start");
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache_at_start();
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type);
let mut vec_file_entry: Vec<FileEntry> = non_cached_files_to_check
.par_iter()
@ -432,18 +334,11 @@ impl SimilarVideos {
hashmap_with_file_entries.insert(file_entry.vhash.src_path().to_string_lossy().to_string(), file_entry.clone());
vector_of_hashes.push(file_entry.vhash.clone());
} else {
self.text_messages.warnings.push(file_entry.error.clone());
self.common_data.text_messages.warnings.push(file_entry.error.clone());
}
}
if self.use_cache {
// Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = loaded_hash_map;
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
save_hashes_to_file(&all_results, &mut self.text_messages, self.save_also_as_json);
}
self.save_cache(vec_file_entry, loaded_hash_map);
// Break if stop was clicked after saving to cache
if check_was_stopped.load(Ordering::Relaxed) {
@ -453,7 +348,7 @@ impl SimilarVideos {
self.match_groups_of_videos(vector_of_hashes, &hashmap_with_file_entries);
self.remove_from_reference_folders();
if self.use_reference_folders {
if self.common_data.use_reference_folders {
for (_fe, vector) in &self.similar_referenced_vectors {
self.information.number_of_duplicates += vector.len();
self.information.number_of_groups += 1;
@ -469,10 +364,24 @@ impl SimilarVideos {
self.videos_hashes = Default::default();
self.videos_to_check = Default::default();
debug!("sort_videos - end");
true
}
fn save_cache(&mut self, vec_file_entry: Vec<FileEntry>, loaded_hash_map: BTreeMap<String, FileEntry>) {
debug!("save_cache - start, use cache: {}", self.common_data.use_cache);
if self.common_data.use_cache {
// Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = loaded_hash_map;
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
save_hashes_to_file(&all_results, &mut self.common_data.text_messages, self.common_data.save_also_as_json);
}
debug!("save_cache - end");
}
fn match_groups_of_videos(&mut self, vector_of_hashes: Vec<VideoHash>, hashmap_with_file_entries: &HashMap<String, FileEntry>) {
debug!("match_groups_of_videos - start");
let match_group = vid_dup_finder_lib::search(vector_of_hashes, NormalizedTolerance::new(self.tolerance as f64 / 100.0f64));
let mut collected_similar_videos: Vec<Vec<FileEntry>> = Default::default();
for i in match_group {
@ -495,15 +404,18 @@ impl SimilarVideos {
}
self.similar_vectors = collected_similar_videos;
debug!("match_groups_of_videos - end");
}
fn remove_from_reference_folders(&mut self) {
if self.use_reference_folders {
debug!("remove_from_reference_folders - start, use reference folders: {}", self.common_data.use_reference_folders);
if self.common_data.use_reference_folders {
self.similar_referenced_vectors = mem::take(&mut self.similar_vectors)
.into_iter()
.filter_map(|vec_file_entry| {
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) =
vec_file_entry.into_iter().partition(|e| self.directories.is_in_referenced_directory(e.get_path()));
let (mut files_from_referenced_folders, normal_files): (Vec<_>, Vec<_>) = vec_file_entry
.into_iter()
.partition(|e| self.common_data.directories.is_in_referenced_directory(e.get_path()));
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
@ -513,23 +425,7 @@ impl SimilarVideos {
})
.collect::<Vec<(FileEntry, Vec<FileEntry>)>>();
}
}
/// Set included dir which needs to be relative, exists etc.
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) {
self.directories.set_included_directory(included_directory, &mut self.text_messages);
}
pub fn set_reference_directory(&mut self, reference_directory: Vec<PathBuf>) {
self.directories.set_reference_directory(reference_directory);
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
debug!("remove_from_reference_folders - end");
}
}
@ -549,7 +445,8 @@ impl DebugPrint for SimilarVideos {
}
println!("---------------DEBUG PRINT---------------");
println!("Included directories - {:?}", self.directories.included_directories);
println!("Included directories - {:?}", self.common_data.directories.included_directories);
self.debug_print_common();
println!("-----------------------------------------");
}
}
@ -564,7 +461,7 @@ impl SaveResults for SimilarVideos {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -573,9 +470,12 @@ impl SaveResults for SimilarVideos {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -682,7 +582,6 @@ fn get_cache_file() -> String {
"cache_similar_videos.bin".to_string()
}
#[must_use]
pub fn check_if_ffmpeg_is_installed() -> bool {
let vid = "9999czekoczekoczekolada999.txt";
if let Err(DetermineVideo {
@ -694,3 +593,38 @@ pub fn check_if_ffmpeg_is_installed() -> bool {
}
true
}
impl SimilarVideos {
pub fn set_exclude_videos_with_same_size(&mut self, exclude_videos_with_same_size: bool) {
self.exclude_videos_with_same_size = exclude_videos_with_same_size;
}
pub fn set_tolerance(&mut self, tolerance: i32) {
assert!((0..=MAX_TOLERANCE).contains(&tolerance));
self.tolerance = tolerance;
}
pub const fn get_similar_videos(&self) -> &Vec<Vec<FileEntry>> {
&self.similar_vectors
}
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn get_similar_videos_referenced(&self) -> &Vec<(FileEntry, Vec<FileEntry>)> {
&self.similar_referenced_vectors
}
pub fn get_number_of_base_duplicated_files(&self) -> usize {
if self.common_data.use_reference_folders {
self.similar_referenced_vectors.len()
} else {
self.similar_vectors.len()
}
}
pub fn get_use_reference(&self) -> bool {
self.common_data.use_reference_folders
}
}

192
czkawka_core/src/temporary.rs
View file

@ -8,13 +8,12 @@ use std::sync::Arc;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use log::{debug, info};
use rayon::prelude::*;
use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_items::ExcludedItems;
use crate::common_messages::Messages;
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::*;
const TEMP_EXTENSIONS: &[&str] = &[
@ -51,108 +50,52 @@ pub struct Info {
pub number_of_temporary_files: usize,
}
impl Info {
#[must_use]
pub fn new() -> Self {
Default::default()
}
}
/// Struct with required information's to work
pub struct Temporary {
tool_type: ToolType,
text_messages: Messages,
common_data: CommonToolData,
information: Info,
temporary_files: Vec<FileEntry>,
directories: Directories,
excluded_items: ExcludedItems,
recursive_search: bool,
delete_method: DeleteMethod,
stopped_search: bool,
}
impl Temporary {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::TemporaryFiles,
text_messages: Messages::new(),
information: Info::new(),
recursive_search: true,
directories: Directories::new(),
excluded_items: ExcludedItems::new(),
common_data: CommonToolData::new(ToolType::TemporaryFiles),
information: Info::default(),
delete_method: DeleteMethod::None,
temporary_files: vec![],
stopped_search: false,
}
}
/// Finding temporary files, save results to internal struct variables
pub fn find_temporary_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.directories.optimize_directories(self.recursive_search, &mut self.text_messages);
fn find_temporary_files_internal(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
self.optimize_dirs_before_start();
if !self.check_files(stop_receiver, progress_sender) {
self.stopped_search = true;
self.common_data.stopped_search = true;
return;
}
self.delete_files();
self.debug_print();
}
#[must_use]
pub fn get_stopped_search(&self) -> bool {
self.stopped_search
}
#[must_use]
pub const fn get_temporary_files(&self) -> &Vec<FileEntry> {
&self.temporary_files
}
#[must_use]
pub const fn get_text_messages(&self) -> &Messages {
&self.text_messages
}
#[must_use]
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
pub fn set_recursive_search(&mut self, recursive_search: bool) {
self.recursive_search = recursive_search;
}
#[cfg(target_family = "unix")]
pub fn set_exclude_other_filesystems(&mut self, exclude_other_filesystems: bool) {
self.directories.set_exclude_other_filesystems(exclude_other_filesystems);
}
#[cfg(not(target_family = "unix"))]
pub fn set_exclude_other_filesystems(&mut self, _exclude_other_filesystems: bool) {}
pub fn set_included_directory(&mut self, included_directory: Vec<PathBuf>) -> bool {
self.directories.set_included_directory(included_directory, &mut self.text_messages)
}
pub fn set_excluded_directory(&mut self, excluded_directory: Vec<PathBuf>) {
self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages);
}
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages);
pub fn find_temporary_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) {
info!("Starting finding temporary files");
let start_time = std::time::Instant::now();
self.find_temporary_files_internal(stop_receiver, progress_sender);
info!("Ended finding temporary files which took {:?}", start_time.elapsed());
}
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
debug!("check_files - start");
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
// Add root folders for finding
for id in &self.directories.included_directories {
for id in &self.common_data.directories.included_directories {
folders_to_check.push(id.clone());
}
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None, self.tool_type);
prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None, self.common_data.tool_type);
while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
@ -183,9 +126,9 @@ impl Temporary {
&mut warnings,
current_folder,
entry_data,
self.recursive_search,
&self.directories,
&self.excluded_items,
self.common_data.recursive_search,
&self.common_data.directories,
&self.common_data.excluded_items,
);
} else if metadata.is_file() {
if let Some(file_entry) = self.get_file_entry(&metadata, &atomic_counter, entry_data, &mut warnings, current_folder) {
@ -203,7 +146,7 @@ impl Temporary {
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.text_messages.warnings.extend(warnings);
self.common_data.text_messages.warnings.extend(warnings);
for fe in fe_result {
self.temporary_files.push(fe);
}
@ -213,6 +156,7 @@ impl Temporary {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.information.number_of_temporary_files = self.temporary_files.len();
debug!("check_files - end");
true
}
pub fn get_file_entry(
@ -233,7 +177,7 @@ impl Temporary {
return None;
}
let current_file_name = current_folder.join(entry_data.file_name());
if self.excluded_items.is_excluded(&current_file_name) {
if self.common_data.excluded_items.is_excluded(&current_file_name) {
return None;
}
@ -248,11 +192,13 @@ impl Temporary {
fn delete_files(&mut self) {
match self.delete_method {
DeleteMethod::Delete => {
let mut warnings = Vec::new();
for file_entry in &self.temporary_files {
if fs::remove_file(file_entry.path.clone()).is_err() {
self.text_messages.warnings.push(file_entry.path.display().to_string());
warnings.push(file_entry.path.display().to_string());
}
}
self.common_data.text_messages.warnings.extend(warnings);
}
DeleteMethod::None => {
//Just do nothing
@ -261,41 +207,6 @@ impl Temporary {
}
}
impl Default for Temporary {
fn default() -> Self {
Self::new()
}
}
impl DebugPrint for Temporary {
#[allow(dead_code)]
#[allow(unreachable_code)]
fn debug_print(&self) {
#[cfg(not(debug_assertions))]
{
return;
}
println!("---------------DEBUG PRINT---------------");
println!("### Information's");
println!("Errors size - {}", self.text_messages.errors.len());
println!("Warnings size - {}", self.text_messages.warnings.len());
println!("Messages size - {}", self.text_messages.messages.len());
println!("### Other");
println!("Temporary list size - {}", self.temporary_files.len());
println!("Excluded items - {:?}", self.excluded_items.items);
println!("Included directories - {:?}", self.directories.included_directories);
println!("Excluded directories - {:?}", self.directories.excluded_directories);
println!("Recursive search - {}", self.recursive_search);
#[cfg(target_family = "unix")]
println!("Skip other filesystems - {}", self.directories.exclude_other_filesystems());
println!("Delete Method - {:?}", self.delete_method);
println!("-----------------------------------------");
}
}
impl SaveResults for Temporary {
fn save_results_to_file(&mut self, file_name: &str) -> bool {
let file_name: String = match file_name {
@ -306,7 +217,7 @@ impl SaveResults for Temporary {
let file_handler = match File::create(&file_name) {
Ok(t) => t,
Err(e) => {
self.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
self.common_data.text_messages.errors.push(format!("Failed to create file {file_name}, reason {e}"));
return false;
}
};
@ -315,9 +226,12 @@ impl SaveResults for Temporary {
if let Err(e) = writeln!(
writer,
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
self.directories.included_directories, self.directories.excluded_directories, self.excluded_items.items
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
) {
self.text_messages.errors.push(format!("Failed to save results to file {file_name}, reason {e}"));
self.common_data
.text_messages
.errors
.push(format!("Failed to save results to file {file_name}, reason {e}"));
return false;
}
@ -342,3 +256,47 @@ impl PrintResults for Temporary {
}
}
}
impl Default for Temporary {
fn default() -> Self {
Self::new()
}
}
impl DebugPrint for Temporary {
#[allow(dead_code)]
#[allow(unreachable_code)]
fn debug_print(&self) {
#[cfg(not(debug_assertions))]
{
return;
}
println!("### Information's");
println!("Temporary list size - {}", self.temporary_files.len());
println!("Delete Method - {:?}", self.delete_method);
self.debug_print_common();
}
}
impl CommonData for Temporary {
fn get_cd(&self) -> &CommonToolData {
&self.common_data
}
fn get_cd_mut(&mut self) -> &mut CommonToolData {
&mut self.common_data
}
}
impl Temporary {
pub const fn get_temporary_files(&self) -> &Vec<FileEntry> {
&self.temporary_files
}
pub const fn get_information(&self) -> &Info {
&self.information
}
pub fn set_delete_method(&mut self, delete_method: DeleteMethod) {
self.delete_method = delete_method;
}
}

30
czkawka_gui/Cargo.toml
View file

@ -3,18 +3,18 @@ name = "czkawka_gui"
version = "6.0.0"
authors = ["Rafał Mikrut <mikrutrafal@protonmail.com>"]
edition = "2021"
rust-version = "1.67.1"
rust-version = "1.70.0"
description = "GTK frontend of Czkawka"
license = "MIT"
homepage = "https://github.com/qarmin/czkawka"
repository = "https://github.com/qarmin/czkawka"
[dependencies]
gdk4 = "0.6"
glib = "0.17"
gdk4 = "0.7"
glib = "0.18"
humansize = "2.1"
chrono = "0.4.26"
chrono = "0.4.31"
# Used for sending stop signal across threads
crossbeam-channel = "0.5.8"
@ -44,24 +44,20 @@ trash = "3.0"
fs_extra = "1.3"
# Language
i18n-embed = { version = "0.13", features = ["fluent-system", "desktop-requester"] }
i18n-embed-fl = "0.6"
rust-embed = "6.8"
i18n-embed = { version = "0.14", features = ["fluent-system", "desktop-requester"] }
i18n-embed-fl = "0.7"
rust-embed = "8.0"
once_cell = "1.18"
log = "0.4.20"
handsome_logger = "0.8"
czkawka_core = { path = "../czkawka_core", version = "6.0.0", features = [] }
gtk4 = { version = "0.7", default-features = false, features = ["v4_6"] }
[target.'cfg(windows)'.dependencies]
winapi = { version = "0.3.9", features = ["combaseapi", "objbase", "shobjidl_core", "windef", "winerror", "wtypesbase", "winuser"] }
[dependencies.gtk4]
version = "0.6"
default-features = false
features = ["v4_6"]
[dependencies.czkawka_core]
path = "../czkawka_core"
version = "6.0.0"
features = []
[features]
default = []
heif = ["czkawka_core/heif"]

13
czkawka_gui/src/compute_results.rs
View file

@ -14,6 +14,7 @@ use czkawka_core::big_file::BigFile;
use czkawka_core::broken_files::BrokenFiles;
use czkawka_core::common::split_path;
use czkawka_core::common_dir_traversal::{CheckingMethod, FileEntry};
use czkawka_core::common_tool::CommonData;
use czkawka_core::duplicate::DuplicateFinder;
use czkawka_core::empty_files::EmptyFiles;
use czkawka_core::empty_folder::EmptyFolder;
@ -224,7 +225,7 @@ pub fn connect_compute_results(gui_data: &GuiData, glib_stop_receiver: Receiver<
}
}
// Returning false here would close the receiver and have senders fail
Continue(true)
glib::ControlFlow::Continue
});
}
@ -659,7 +660,7 @@ fn computer_similar_videos(
if ff.get_use_reference() {
let vec_struct_similar = ff.get_similar_videos_referenced();
for (base_file_entry, vec_file_entry) in vec_struct_similar.iter() {
for (base_file_entry, vec_file_entry) in vec_struct_similar {
// Sort
let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone();
@ -681,7 +682,7 @@ fn computer_similar_videos(
} else {
let vec_struct_similar = ff.get_similar_videos();
for vec_file_entry in vec_struct_similar.iter() {
for vec_file_entry in vec_struct_similar {
// Sort
let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone();
@ -761,7 +762,7 @@ fn computer_similar_images(
if sf.get_use_reference() {
let vec_struct_similar: &Vec<(similar_images::FileEntry, Vec<similar_images::FileEntry>)> = sf.get_similar_images_referenced();
for (base_file_entry, vec_file_entry) in vec_struct_similar.iter() {
for (base_file_entry, vec_file_entry) in vec_struct_similar {
// Sort
let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone();
@ -804,7 +805,7 @@ fn computer_similar_images(
}
} else {
let vec_struct_similar = sf.get_similar_images();
for vec_file_entry in vec_struct_similar.iter() {
for vec_file_entry in vec_struct_similar {
// Sort
let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone();
@ -956,7 +957,7 @@ fn computer_big_files(
let vector = bf.get_big_files();
for (size, file_entry) in vector.iter() {
for (size, file_entry) in vector {
let (directory, file) = split_path(&file_entry.path);
let values: [(u32, &dyn ToValue); COLUMNS_NUMBER] = [
(ColumnsBigFiles::SelectionButton as u32, &false),

2
czkawka_gui/src/connect_things/connect_button_compare.rs
View file

@ -104,7 +104,7 @@ pub fn connect_button_compare(gui_data: &GuiData) {
*shared_using_for_preview.borrow_mut() = (None, None);
image_compare_left.set_from_pixbuf(None);
image_compare_right.set_from_pixbuf(None);
gtk4::Inhibit(true)
glib::Propagation::Stop
});
let button_go_previous_compare_group = gui_data.compare_images.button_go_previous_compare_group.clone();

10
czkawka_gui/src/connect_things/connect_button_delete.rs
View file

@ -353,14 +353,14 @@ pub fn empty_folder_remover(
if !error_happened {
if !use_trash {
match fs::remove_dir_all(get_full_name_from_path_name(&path, &name)) {
Ok(_) => {
Ok(()) => {
model.remove(&iter);
}
Err(_inspected) => error_happened = true,
}
} else {
match trash::delete(get_full_name_from_path_name(&path, &name)) {
Ok(_) => {
Ok(()) => {
model.remove(&iter);
}
Err(_inspected) => error_happened = true,
@ -420,7 +420,7 @@ pub fn basic_remove(
if !use_trash {
match fs::remove_file(get_full_name_from_path_name(&path, &name)) {
Ok(_) => {
Ok(()) => {
model.remove(&iter);
}
@ -435,7 +435,7 @@ pub fn basic_remove(
}
} else {
match trash::delete(get_full_name_from_path_name(&path, &name)) {
Ok(_) => {
Ok(()) => {
model.remove(&iter);
}
Err(e) => {
@ -503,7 +503,7 @@ pub fn tree_remove(
model.remove(&iter);
map_with_path_to_delete.entry(path.clone()).or_insert_with(Vec::new).push(file_name);
map_with_path_to_delete.entry(path.clone()).or_default().push(file_name);
}
// Delete duplicated entries, and remove real files

45
czkawka_gui/src/connect_things/connect_button_search.rs
View file

@ -1,18 +1,20 @@
use crossbeam_channel::Receiver;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
use crossbeam_channel::Receiver;
use futures::channel::mpsc::UnboundedSender;
use glib::Sender;
use gtk4::prelude::*;
use gtk4::Grid;
use log::debug;
use czkawka_core::bad_extensions::BadExtensions;
use czkawka_core::big_file::BigFile;
use czkawka_core::broken_files::{BrokenFiles, CheckedTypes};
use czkawka_core::common_dir_traversal::{CheckingMethod, ProgressData};
use czkawka_core::common_tool::CommonData;
use czkawka_core::duplicate::DuplicateFinder;
use czkawka_core::empty_files::EmptyFiles;
use czkawka_core::empty_folder::EmptyFolder;
@ -84,7 +86,7 @@ pub fn connect_button_search(gui_data: &GuiData, glib_stop_sender: Sender<Messag
let glib_stop_sender = glib_stop_sender.clone();
let stop_receiver = stop_receiver.clone();
// Consume any stale stop messages.
stop_receiver.try_iter().for_each(|_| ());
stop_receiver.try_iter().for_each(|()| ());
label_stage.show();
@ -207,6 +209,7 @@ struct LoadedCommonItems {
maximal_file_size: u64,
ignore_other_filesystems: bool,
}
impl LoadedCommonItems {
fn load_items(gui_data: &GuiData) -> Self {
let check_button_settings_one_filesystem = gui_data.settings.check_button_settings_one_filesystem.clone();
@ -276,6 +279,7 @@ impl LoadedCommonItems {
}
}
}
fn duplicate_search(
gui_data: &GuiData,
loaded_common_items: LoadedCommonItems,
@ -296,7 +300,7 @@ fn duplicate_search(
let tree_view_duplicate_finder = gui_data.main_notebook.tree_view_duplicate_finder.clone();
image_preview_duplicates.hide();
get_list_store(&tree_view_duplicate_finder).clear();
clean_tree_view(&tree_view_duplicate_finder);
let check_method_index = combo_box_duplicate_check_method.active().unwrap() as usize;
let check_method = DUPLICATES_CHECK_METHOD_COMBO_BOX[check_method_index].check_method;
@ -348,7 +352,7 @@ fn empty_files_search(
grid_progress_stages.hide();
let tree_view_empty_files_finder = gui_data.main_notebook.tree_view_empty_files_finder.clone();
get_list_store(&tree_view_empty_files_finder).clear();
clean_tree_view(&tree_view_empty_files_finder);
// Find empty files
thread::spawn(move || {
let mut vf = EmptyFiles::new();
@ -375,7 +379,7 @@ fn empty_directories_search(
grid_progress_stages.hide();
let tree_view_empty_folder_finder = gui_data.main_notebook.tree_view_empty_folder_finder.clone();
get_list_store(&tree_view_empty_folder_finder).clear();
clean_tree_view(&tree_view_empty_folder_finder);
thread::spawn(move || {
let mut ef = EmptyFolder::new();
@ -387,6 +391,7 @@ fn empty_directories_search(
glib_stop_sender.send(Message::EmptyFolders(ef)).unwrap();
});
}
fn big_files_search(
gui_data: &GuiData,
loaded_common_items: LoadedCommonItems,
@ -400,7 +405,7 @@ fn big_files_search(
let combo_box_big_files_mode = gui_data.main_notebook.combo_box_big_files_mode.clone();
let entry_big_files_number = gui_data.main_notebook.entry_big_files_number.clone();
let tree_view_big_files_finder = gui_data.main_notebook.tree_view_big_files_finder.clone();
get_list_store(&tree_view_big_files_finder).clear();
clean_tree_view(&tree_view_big_files_finder);
let big_files_mode_index = combo_box_big_files_mode.active().unwrap() as usize;
let big_files_mode = BIG_FILES_CHECK_METHOD_COMBO_BOX[big_files_mode_index].check_method;
@ -422,6 +427,7 @@ fn big_files_search(
glib_stop_sender.send(Message::BigFiles(bf)).unwrap();
});
}
fn temporary_files_search(
gui_data: &GuiData,
loaded_common_items: LoadedCommonItems,
@ -433,7 +439,7 @@ fn temporary_files_search(
grid_progress_stages.hide();
let tree_view_temporary_files_finder = gui_data.main_notebook.tree_view_temporary_files_finder.clone();
get_list_store(&tree_view_temporary_files_finder).clear();
clean_tree_view(&tree_view_temporary_files_finder);
thread::spawn(move || {
let mut tf = Temporary::new();
@ -447,6 +453,7 @@ fn temporary_files_search(
glib_stop_sender.send(Message::Temporary(tf)).unwrap();
});
}
fn same_music_search(
gui_data: &GuiData,
loaded_common_items: LoadedCommonItems,
@ -470,7 +477,7 @@ fn same_music_search(
let scale_seconds_same_music = gui_data.main_notebook.scale_seconds_same_music.clone();
let scale_similarity_same_music = gui_data.main_notebook.scale_similarity_same_music.clone();
get_list_store(&tree_view_same_music_finder).clear();
clean_tree_view(&tree_view_same_music_finder);
let approximate_comparison = check_button_music_approximate_comparison.is_active();
@ -548,6 +555,7 @@ fn same_music_search(
button_app_info.set_sensitive(true);
}
}
fn broken_files_search(
gui_data: &GuiData,
loaded_common_items: LoadedCommonItems,
@ -565,7 +573,7 @@ fn broken_files_search(
let check_button_broken_files_image: gtk4::CheckButton = gui_data.main_notebook.check_button_broken_files_image.clone();
let tree_view_broken_files = gui_data.main_notebook.tree_view_broken_files.clone();
get_list_store(&tree_view_broken_files).clear();
clean_tree_view(&tree_view_broken_files);
let mut checked_types: CheckedTypes = CheckedTypes::NONE;
@ -622,6 +630,7 @@ fn broken_files_search(
button_app_info.set_sensitive(true);
}
}
fn similar_image_search(
gui_data: &GuiData,
loaded_common_items: LoadedCommonItems,
@ -641,7 +650,7 @@ fn similar_image_search(
let scale_similarity_similar_images = gui_data.main_notebook.scale_similarity_similar_images.clone();
let tree_view_similar_images_finder = gui_data.main_notebook.tree_view_similar_images_finder.clone();
get_list_store(&tree_view_similar_images_finder).clear();
clean_tree_view(&tree_view_similar_images_finder);
image_preview_similar_images.hide();
let hash_size_index = combo_box_image_hash_size.active().unwrap() as usize;
@ -683,6 +692,7 @@ fn similar_image_search(
glib_stop_sender.send(Message::SimilarImages(sf)).unwrap();
});
}
fn similar_video_search(
gui_data: &GuiData,
loaded_common_items: LoadedCommonItems,
@ -697,7 +707,7 @@ fn similar_video_search(
let check_button_settings_similar_videos_delete_outdated_cache = gui_data.settings.check_button_settings_similar_videos_delete_outdated_cache.clone();
let scale_similarity_similar_videos = gui_data.main_notebook.scale_similarity_similar_videos.clone();
let tree_view_similar_videos_finder = gui_data.main_notebook.tree_view_similar_videos_finder.clone();
get_list_store(&tree_view_similar_videos_finder).clear();
clean_tree_view(&tree_view_similar_videos_finder);
let tolerance = scale_similarity_similar_videos.value() as i32;
@ -726,6 +736,7 @@ fn similar_video_search(
glib_stop_sender.send(Message::SimilarVideos(sf)).unwrap();
});
}
fn bad_symlinks_search(
gui_data: &GuiData,
loaded_common_items: LoadedCommonItems,
@ -737,7 +748,7 @@ fn bad_symlinks_search(
grid_progress_stages.hide();
let tree_view_invalid_symlinks = gui_data.main_notebook.tree_view_invalid_symlinks.clone();
get_list_store(&tree_view_invalid_symlinks).clear();
clean_tree_view(&tree_view_invalid_symlinks);
thread::spawn(move || {
let mut isf = InvalidSymlinks::new();
@ -752,6 +763,7 @@ fn bad_symlinks_search(
glib_stop_sender.send(Message::InvalidSymlinks(isf)).unwrap();
});
}
fn bad_extensions_search(
gui_data: &GuiData,
loaded_common_items: LoadedCommonItems,
@ -763,7 +775,7 @@ fn bad_extensions_search(
grid_progress_stages.show();
let tree_view_bad_extensions = gui_data.main_notebook.tree_view_bad_extensions.clone();
get_list_store(&tree_view_bad_extensions).clear();
clean_tree_view(&tree_view_bad_extensions);
thread::spawn(move || {
let mut be = BadExtensions::new();
@ -780,3 +792,10 @@ fn bad_extensions_search(
glib_stop_sender.send(Message::BadExtensions(be)).unwrap();
});
}
fn clean_tree_view(tree_view: &gtk4::TreeView) {
debug!("Start clean tree view");
let list_store = get_list_store(tree_view);
list_store.clear();
debug!("Cleared tree view");
}

2
czkawka_gui/src/connect_things/connect_button_stop.rs
View file

@ -8,7 +8,7 @@ use crate::help_functions::KEY_ENTER;
fn send_stop_message(stop_sender: &Sender<()>) {
stop_sender
.try_send(())
.map_or_else(|e| if matches!(e, TrySendError::Full(_)) { Ok(()) } else { Err(e) }, |_| Ok(()))
.map_or_else(|e| if matches!(e, TrySendError::Full(())) { Ok(()) } else { Err(e) }, |()| Ok(()))
.unwrap();
}

3
czkawka_gui/src/connect_things/connect_header_buttons.rs
View file

@ -1,5 +1,4 @@
use gtk4::prelude::*;
use gtk4::Inhibit;
use crate::gui_structs::gui_data::GuiData;
@ -12,7 +11,7 @@ pub fn connect_button_about(gui_data: &GuiData) {
// Prevent from deleting dialog after close
about_dialog.connect_close_request(|dialog| {
dialog.hide();
Inhibit(true)
glib::Propagation::Stop
});
});
}

12
czkawka_gui/src/connect_things/connect_progress_window.rs
View file

@ -44,6 +44,7 @@ pub fn connect_progress_window(gui_data: &GuiData, mut progress_receiver: Unboun
};
main_context.spawn_local(future);
}
fn process_bar_empty_files(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
@ -51,6 +52,7 @@ fn process_bar_empty_files(gui_data: &GuiData, item: &ProgressData) {
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
fn process_bar_empty_folder(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
@ -61,6 +63,7 @@ fn process_bar_empty_folder(gui_data: &GuiData, item: &ProgressData) {
));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
fn process_bar_big_files(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
@ -68,6 +71,7 @@ fn process_bar_big_files(gui_data: &GuiData, item: &ProgressData) {
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
fn process_bar_same_music(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
@ -110,6 +114,7 @@ fn process_bar_same_music(gui_data: &GuiData, item: &ProgressData) {
_ => panic!(),
}
}
fn process_bar_similar_images(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
@ -135,6 +140,7 @@ fn process_bar_similar_images(gui_data: &GuiData, item: &ProgressData) {
_ => panic!(),
}
}
fn process_bar_similar_videos(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
@ -155,6 +161,7 @@ fn process_bar_similar_videos(gui_data: &GuiData, item: &ProgressData) {
_ => panic!(),
}
}
fn process_bar_temporary(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
@ -162,6 +169,7 @@ fn process_bar_temporary(gui_data: &GuiData, item: &ProgressData) {
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
fn process_bar_invalid_symlinks(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let taskbar_state = gui_data.taskbar_state.clone();
@ -169,6 +177,7 @@ fn process_bar_invalid_symlinks(gui_data: &GuiData, item: &ProgressData) {
label_stage.set_text(&flg!("progress_scanning_general_file", file_number_tm(item)));
taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE);
}
fn process_bar_broken_files(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
@ -189,6 +198,7 @@ fn process_bar_broken_files(gui_data: &GuiData, item: &ProgressData) {
_ => panic!(),
}
}
fn process_bar_bad_extensions(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
@ -209,6 +219,7 @@ fn process_bar_bad_extensions(gui_data: &GuiData, item: &ProgressData) {
_ => panic!(),
}
}
fn process_bar_duplicates(gui_data: &GuiData, item: &ProgressData) {
let label_stage = gui_data.progress_window.label_stage.clone();
let progress_bar_current_stage = gui_data.progress_window.progress_bar_current_stage.clone();
@ -289,6 +300,7 @@ fn common_set_data(item: &ProgressData, progress_bar_all_stages: &ProgressBar, p
fn file_number_tm(item: &ProgressData) -> HashMap<&'static str, String> {
generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())])
}
fn progress_ratio_tm(item: &ProgressData) -> HashMap<&'static str, String> {
generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())])
}

2
czkawka_gui/src/connect_things/connect_settings.rs
View file

@ -37,7 +37,7 @@ pub fn connect_settings(gui_data: &GuiData) {
window_settings.connect_close_request(move |window| {
window.hide();
gtk4::Inhibit(true)
glib::Propagation::Stop
});
}

7
czkawka_gui/src/help_functions.rs
View file

@ -4,7 +4,6 @@ use std::io::BufReader;
use std::path::PathBuf;
use gdk4::gdk_pixbuf::{InterpType, Pixbuf};
use glib::signal::Inhibit;
use glib::Error;
use gtk4::prelude::*;
use gtk4::{ListStore, Scale, ScrollType, TextView, TreeView, Widget};
@ -378,6 +377,7 @@ pub fn get_notebook_enum_from_tree_view(tree_view: &TreeView) -> NotebookMainEnu
}
}
}
pub fn get_tree_view_name_from_notebook_enum(notebook_enum: NotebookMainEnum) -> &'static str {
match notebook_enum {
NotebookMainEnum::Duplicate => "tree_view_duplicate_finder",
@ -401,6 +401,7 @@ pub fn get_notebook_upper_enum_from_tree_view(tree_view: &TreeView) -> NotebookU
e => panic!("{}", e),
}
}
pub fn get_tree_view_name_from_notebook_upper_enum(notebook_upper_enum: NotebookUpperEnum) -> &'static str {
match notebook_upper_enum {
NotebookUpperEnum::IncludedDirectories => "tree_view_upper_included_directories",
@ -799,11 +800,11 @@ pub fn scale_set_min_max_values(scale: &Scale, minimum: f64, maximum: f64, curre
}
}
pub fn scale_step_function(scale: &Scale, _scroll_type: ScrollType, value: f64) -> Inhibit {
pub fn scale_step_function(scale: &Scale, _scroll_type: ScrollType, value: f64) -> glib::Propagation {
scale.set_increments(1_f64, 1_f64);
scale.set_round_digits(0);
scale.set_fill_level(value.round());
Inhibit(false)
glib::Propagation::Proceed
}
#[cfg(test)]

4
czkawka_gui/src/initialize_gui.rs
View file

@ -27,7 +27,7 @@ use crate::notebook_info::NOTEBOOKS_INFO;
use crate::opening_selecting_records::*;
use crate::{delete_things, flg};
pub fn initialize_gui(gui_data: &mut GuiData) {
pub fn initialize_gui(gui_data: &GuiData) {
//// Initialize button
{
let buttons = &gui_data.bottom_buttons.buttons_array;
@ -273,7 +273,7 @@ pub fn initialize_gui(gui_data: &mut GuiData) {
window_progress.connect_close_request(move |_| {
stop_sender.send(()).unwrap();
gtk4::Inhibit(true)
glib::Propagation::Stop
});
}

14
czkawka_gui/src/main.rs
View file

@ -10,9 +10,10 @@ use std::ffi::OsString;
use futures::channel::mpsc;
use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender};
use glib::Priority;
use gtk4::gio::ApplicationFlags;
use gtk4::prelude::*;
use gtk4::{Application, Inhibit};
use gtk4::Application;
use connect_things::connect_about_buttons::*;
use connect_things::connect_button_compare::*;
@ -32,7 +33,7 @@ use connect_things::connect_selection_of_directories::*;
use connect_things::connect_settings::*;
use connect_things::connect_show_hide_ui::*;
use connect_things::connect_similar_image_size_change::*;
use czkawka_core::common::{get_number_of_threads, set_number_of_threads};
use czkawka_core::common::{get_number_of_threads, set_number_of_threads, setup_logger};
use czkawka_core::common_dir_traversal::ProgressData;
use czkawka_core::*;
use gui_structs::gui_data::*;
@ -70,6 +71,7 @@ mod tests;
fn main() {
let application = Application::new(None::<String>, ApplicationFlags::HANDLES_OPEN | ApplicationFlags::HANDLES_COMMAND_LINE);
application.connect_command_line(move |app, cmdline| {
setup_logger(false);
build_ui(app, &cmdline.arguments());
0
});
@ -77,15 +79,15 @@ fn main() {
}
fn build_ui(application: &Application, arguments: &[OsString]) {
let mut gui_data: GuiData = GuiData::new_with_application(application);
let gui_data: GuiData = GuiData::new_with_application(application);
// Used for getting data from thread
let (glib_stop_sender, glib_stop_receiver) = glib::MainContext::channel(glib::PRIORITY_DEFAULT);
let (glib_stop_sender, glib_stop_receiver) = glib::MainContext::channel(Priority::default());
// Futures progress report
let (progress_sender, progress_receiver): (UnboundedSender<ProgressData>, UnboundedReceiver<ProgressData>) = mpsc::unbounded();
initialize_gui(&mut gui_data);
initialize_gui(&gui_data);
validate_notebook_data(&gui_data); // Must be run after initialization of gui, to check if everything was properly setup
reset_configuration(false, &gui_data.upper_notebook, &gui_data.main_notebook, &gui_data.settings, &gui_data.text_view_errors); // Fallback for invalid loading setting project
load_system_language(&gui_data); // Check for default system language, must be loaded after initializing GUI and before loading settings from file
@ -138,6 +140,6 @@ fn build_ui(application: &Application, arguments: &[OsString]) {
// Save configuration at exit
}
taskbar_state.borrow_mut().release();
Inhibit(false)
glib::Propagation::Proceed
});
}

14
czkawka_gui/src/opening_selecting_records.rs
View file

@ -1,5 +1,4 @@
use gdk4::{Key, ModifierType};
use glib::signal::Inhibit;
use gtk4::prelude::*;
use gtk4::GestureClick;
@ -8,7 +7,12 @@ use crate::notebook_enums::NotebookUpperEnum;
// TODO add option to open files and folders from context menu activated by pressing ONCE with right mouse button
pub fn opening_enter_function_ported_upper_directories(event_controller: &gtk4::EventControllerKey, _key_value: Key, key_code: u32, _modifier_type: ModifierType) -> Inhibit {
pub fn opening_enter_function_ported_upper_directories(
event_controller: &gtk4::EventControllerKey,
_key_value: Key,
key_code: u32,
_modifier_type: ModifierType,
) -> glib::Propagation {
let tree_view = event_controller.widget().downcast::<gtk4::TreeView>().unwrap();
#[cfg(debug_assertions)]
{
@ -32,7 +36,7 @@ pub fn opening_enter_function_ported_upper_directories(event_controller: &gtk4::
}
}
// false // True catches signal, and don't send it to function, e.g. up button is caught and don't move selection
Inhibit(false)
glib::Propagation::Proceed
}
pub fn opening_middle_mouse_function(gesture_click: &GestureClick, _number_of_clicks: i32, _b: f64, _c: f64) {
@ -64,7 +68,7 @@ pub fn opening_double_click_function_directories(gesture_click: &GestureClick, n
}
}
pub fn opening_enter_function_ported(event_controller: &gtk4::EventControllerKey, _key: Key, key_code: u32, _modifier_type: ModifierType) -> Inhibit {
pub fn opening_enter_function_ported(event_controller: &gtk4::EventControllerKey, _key: Key, key_code: u32, _modifier_type: ModifierType) -> glib::Propagation {
let tree_view = event_controller.widget().downcast::<gtk4::TreeView>().unwrap();
#[cfg(debug_assertions)]
{
@ -80,7 +84,7 @@ pub fn opening_enter_function_ported(event_controller: &gtk4::EventControllerKey
nt_object.column_selection,
nt_object.column_header,
);
Inhibit(false) // True catches signal, and don't send it to function, e.g. up button is caught and don't move selection
glib::Propagation::Proceed // True catches signal, and don't send it to function, e.g. up button is caught and don't move selection
}
pub fn opening_double_click_function(gesture_click: &GestureClick, number_of_clicks: i32, _b: f64, _c: f64) {

4
czkawka_gui/src/saving_loading.rs
View file

@ -316,7 +316,7 @@ impl LoadSaveStruct {
if line.starts_with("--") {
header = line.to_string();
} else if !header.is_empty() {
self.loaded_items.entry(header.clone()).or_insert_with(Vec::new).push(line.to_string());
self.loaded_items.entry(header.clone()).or_default().push(line.to_string());
} else {
add_text_to_text_view(
text_view_errors,
@ -1032,7 +1032,7 @@ pub fn reset_configuration(manual_clearing: bool, upper_notebook: &GuiUpperNoteb
let tree_view_excluded_directories = upper_notebook.tree_view_excluded_directories.clone();
let list_store = get_list_store(&tree_view_excluded_directories);
list_store.clear();
for i in DEFAULT_EXCLUDED_DIRECTORIES.iter() {
for i in DEFAULT_EXCLUDED_DIRECTORIES {
let values: [(u32, &dyn ToValue); 1] = [(ColumnsExcludedDirectory::Path as u32, &i)];
list_store.set(&list_store.append(), &values);
}

14
czkawka_slint_gui/Cargo.toml Normal file
View file

@ -0,0 +1,14 @@
[package]
name = "czkawka_slint"
version = "6.0.0"
authors = ["Rafał Mikrut <mikrutrafal@protonmail.com>"]
edition = "2021"
rust-version = "1.72.1"
description = "Slint frontend of Czkawka"
license = "GPL-3"
homepage = "https://github.com/qarmin/czkawka"
repository = "https://github.com/qarmin/czkawka"
[dependencies]
slint = "1.2.2"
rand = "0.8.5"

232
czkawka_slint_gui/LICENSE Normal file
View file

@ -0,0 +1,232 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright © 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
Preamble
The GNU General Public License is a free, copyleft license for software and other kinds of works.
The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too.
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others.
For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it.
For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions.
Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and modification follow.
TERMS AND CONDITIONS
0. Definitions.
“This License” refers to version 3 of the GNU General Public License.
“Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
“The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations.
To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work.
A “covered work” means either the unmodified Program or a work based on the Program.
To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
1. Source Code.
The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work.
A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.
The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
The Corresponding Source for a work in source code form is that same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”.
c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
“Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
7. Additional Terms.
“Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
11. Patents.
A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's “contributor version”.
A contributor's “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation.
If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:
Czkawka Slint Gui Copyright (C) 2023 Rafał Mikrut
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an “about box”.
You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see <http://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read <http://www.gnu.org/philosophy/why-not-lgpl.html>.

3
czkawka_slint_gui/src/main.rs Normal file
View file

@ -0,0 +1,3 @@
fn main() {}
slint::slint! {}

2
instructions/Compilation.md
View file

@ -12,7 +12,7 @@ New versions of GTK fixes some bugs, so e.g. middle button selection will work o
| Program | Min | What for |
|---------|--------|--------------------------------------------------------------------------------------|
| Rust | 1.67.1 | The minimum version of rust does not depend on anything, so it can change frequently |
| Rust | 1.70.0 | The minimum version of rust does not depend on anything, so it can change frequently |
| GTK | 4.6 | Only for the `GTK` backend |
#### Debian / Ubuntu