From b08cedd5a642191196b9f095410733a63bdfce42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mikrut?= <41945903+qarmin@users.noreply.github.com> Date: Thu, 6 May 2021 21:54:44 +0200 Subject: [PATCH] Allow to set minimal file size to caching (#321) --- Cargo.lock | 76 ++++++++++++------------ czkawka_cli/src/commands.rs | 4 +- czkawka_cli/src/main.rs | 2 + czkawka_core/src/duplicate.rs | 12 +++- czkawka_gui/czkawka.glade | 41 +++++++++++++ czkawka_gui/src/connect_button_search.rs | 3 + czkawka_gui/src/gui_settings.rs | 3 + czkawka_gui/src/saving_loading.rs | 21 +++++++ 8 files changed, 120 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0076922..218d290 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,9 +127,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "base64" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bindgen" @@ -433,9 +433,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" +checksum = "52fb27eab85b17fbb9f6fd667089e07d6a2eb8743d02639ee7f6a7a7729c9c94" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -446,9 +446,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" dependencies = [ "autocfg", "cfg-if 1.0.0", @@ -609,14 +609,14 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "flate2" -version = "1.0.14" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "crc32fast", "libc", - "miniz_oxide 0.3.7", + "miniz_oxide 0.4.4", ] [[package]] @@ -1035,9 +1035,9 @@ dependencies = [ [[package]] name = "img_hash" -version = "3.1.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52057a4b155b3cc36b9cd5b20faaa60f506e39727efcffad7d090f0163fe0c6d" +checksum = "5ea4eac6fc4f64ed363d5c210732b747bfa5ddd8a25ac347d887f298c3a70b49" dependencies = [ "base64", "image", @@ -1086,9 +1086,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "jobserver" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" +checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd" dependencies = [ "libc", ] @@ -1136,9 +1136,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" [[package]] name = "libloading" @@ -1152,9 +1152,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] @@ -1179,9 +1179,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "memoffset" @@ -1459,9 +1459,9 @@ checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" [[package]] name = "open" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7e9f1bdf15cd1f5a00cc9002a733a6ee6d0ff562491852d59652471c4a389f7" +checksum = "1711eb4b31ce4ad35b0f316d8dfba4fe5c7ad601c448446d84aae7a896627b20" dependencies = [ "which", "winapi", @@ -1691,9 +1691,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ "bitflags", ] @@ -1710,18 +1710,18 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.5" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b" dependencies = [ "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.6.23" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "remove_dir_all" @@ -1822,9 +1822,9 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] name = "slab" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" [[package]] name = "slice-deque" @@ -1917,9 +1917,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.69" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" dependencies = [ "proc-macro2", "quote", @@ -2074,9 +2074,9 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "vec_map" @@ -2179,9 +2179,9 @@ dependencies = [ [[package]] name = "weezl" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a32b378380f4e9869b22f0b5177c68a5519f03b3454fde0b291455ddbae266c" +checksum = "d8b77fdfd5a253be4ab714e4ffa3c49caf146b4de743e97510c0656cf90f1e8e" [[package]] name = "which" @@ -2232,9 +2232,9 @@ checksum = "e575e15bedf6e57b5c2d763ffc6c3c760143466cbd09d762d539680ab5992ded" [[package]] name = "zip" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8264fcea9b7a036a4a5103d7153e988dbc2ebbafb34f68a3c2d404b6b82d74b6" +checksum = "9c83dc9b784d252127720168abd71ea82bf8c3d96b17dc565b5e2a02854f2b27" dependencies = [ "byteorder", "bzip2", diff --git a/czkawka_cli/src/commands.rs b/czkawka_cli/src/commands.rs index 9f18a98..0fe3a11 100644 --- a/czkawka_cli/src/commands.rs +++ b/czkawka_cli/src/commands.rs @@ -17,9 +17,11 @@ pub enum Commands { excluded_items: ExcludedItems, #[structopt(short, long, parse(try_from_str = parse_minimal_file_size), default_value = "1024", help = "Minimum size in bytes", long_help = "Minimum size of checked files in bytes, assigning bigger value may speed up searching")] minimal_file_size: u64, + #[structopt(short = "c", long, parse(try_from_str = parse_minimal_file_size), default_value = "2097152", help = "Minimum cached file size in bytes", long_help = "Minimum size of cached files in bytes, assigning bigger value may speed up will cause that lower amount of files will be cached, but loading of cache will be faster")] + minimal_cached_file_size: u64, #[structopt(flatten)] allowed_extensions: AllowedExtensions, - #[structopt(short, long, default_value = "HASH", parse(try_from_str = parse_checking_method), help = "Search method (NAME, SIZE, HASH, HASHMB)", long_help = "Methods to search files.\nNAME - Fast but but rarely usable,\nSIZE - Fast but not accurate, checking by the file's size,\nHASHMB - More accurate but slower, checking by the hash of the file's first mebibyte or\nHASH - The slowest method, checking by the hash of the entire file")] + #[structopt(short, long, default_value = "HASH", parse(try_from_str = parse_checking_method), help = "Search method (NAME, SIZE, HASH, HASHMB)", long_help = "Methods to search files.\nNAME - Fast but but rarely usable,\nSIZE - Fast but not accurate, checking by the file's size,\nHASHMB - More accurate but slower, checking by the hash of the file's first mebibyte\nHASH - The slowest method, checking by the hash of the entire file")] search_method: CheckingMethod, #[structopt(short = "D", long, default_value = "NONE", parse(try_from_str = parse_delete_method), help = "Delete method (AEN, AEO, ON, OO, HARD)", long_help = "Methods to delete the files.\nAEN - All files except the newest,\nAEO - All files except the oldest,\nON - Only 1 file, the newest,\nOO - Only 1 file, the oldest\nHARD - create hard link\nNONE - not delete files")] delete_method: DeleteMethod, diff --git a/czkawka_cli/src/main.rs b/czkawka_cli/src/main.rs index 0b82308..1e6b9c8 100644 --- a/czkawka_cli/src/main.rs +++ b/czkawka_cli/src/main.rs @@ -33,6 +33,7 @@ fn main() { excluded_directories, excluded_items, minimal_file_size, + minimal_cached_file_size, allowed_extensions, search_method, delete_method, @@ -48,6 +49,7 @@ fn main() { df.set_excluded_directory(excluded_directories.excluded_directories); df.set_excluded_items(excluded_items.excluded_items); df.set_minimal_file_size(minimal_file_size); + df.set_minimal_cache_file_size(minimal_cached_file_size); df.set_allowed_extensions(allowed_extensions.allowed_extensions.join(",")); df.set_check_method(search_method); df.set_delete_method(delete_method); diff --git a/czkawka_core/src/duplicate.rs b/czkawka_core/src/duplicate.rs index e6e772f..672db24 100644 --- a/czkawka_core/src/duplicate.rs +++ b/czkawka_core/src/duplicate.rs @@ -152,6 +152,7 @@ pub struct DuplicateFinder { dryrun: bool, stopped_search: bool, use_cache: bool, + minimal_cache_file_size: u64, } impl DuplicateFinder { @@ -174,6 +175,7 @@ impl DuplicateFinder { hash_type: HashType::Blake3, dryrun: false, use_cache: true, + minimal_cache_file_size: 2 * 1024 * 1024, // By default cache only >= 1MB files } } @@ -219,6 +221,10 @@ impl DuplicateFinder { self.stopped_search } + pub fn set_minimal_cache_file_size(&mut self, minimal_cache_file_size: u64) { + self.minimal_cache_file_size = minimal_cache_file_size; + } + pub const fn get_files_sorted_by_names(&self) -> &BTreeMap> { &self.files_with_identical_names } @@ -909,7 +915,7 @@ impl DuplicateFinder { } } } - save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type); + save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type, self.minimal_cache_file_size); } } _ => panic!("What"), @@ -1328,7 +1334,7 @@ pub fn make_hard_link(src: &Path, dst: &Path) -> io::Result<()> { result } -fn save_hashes_to_file(hashmap: &HashMap, text_messages: &mut Messages, type_of_hash: &HashType) { +fn save_hashes_to_file(hashmap: &HashMap, text_messages: &mut Messages, type_of_hash: &HashType, minimal_cache_file_size: u64) { if let Some(proj_dirs) = ProjectDirs::from("pl", "Qarmin", "Czkawka") { let cache_dir = PathBuf::from(proj_dirs.cache_dir()); if cache_dir.exists() { @@ -1352,7 +1358,7 @@ fn save_hashes_to_file(hashmap: &HashMap, text_messages: &mut for file_entry in hashmap.values() { // Only cache bigger than 5MB files - if file_entry.size > 5 * 1024 * 1024 { + if file_entry.size >= minimal_cache_file_size { let string: String = format!("{}//{}//{}//{}", file_entry.path.display(), file_entry.size, file_entry.modified_date, file_entry.hash); if writeln!(writer, "{}", string).is_err() { diff --git a/czkawka_gui/czkawka.glade b/czkawka_gui/czkawka.glade index 7b7d3ab..a6f04bc 100644 --- a/czkawka_gui/czkawka.glade +++ b/czkawka_gui/czkawka.glade @@ -2614,6 +2614,47 @@ This program is free to use and will always be. 0 + + + True + False + 4 + 4 + + + True + False + Minimal cached file size in bytes + + + True + True + 0 + + + + + True + True + 15 + 2097152 + False + number + + + False + False + end + 1 + + + + + False + True + 2 + + 1 diff --git a/czkawka_gui/src/connect_button_search.rs b/czkawka_gui/src/connect_button_search.rs index cd714be..2fc692f 100644 --- a/czkawka_gui/src/connect_button_search.rs +++ b/czkawka_gui/src/connect_button_search.rs @@ -92,6 +92,7 @@ pub fn connect_button_search( let radio_button_hash_type_xxh3 = gui_data.main_notebook.radio_button_hash_type_xxh3.clone(); let check_button_settings_hide_hard_links = gui_data.settings.check_button_settings_hide_hard_links.clone(); let check_button_settings_use_cache = gui_data.settings.check_button_settings_use_cache.clone(); + let entry_settings_cache_file_minimal_size = gui_data.settings.entry_settings_cache_file_minimal_size.clone(); buttons_search_clone.connect_clicked(move |_| { let included_directories = get_path_buf_from_vector_of_strings(get_string_from_list_store(&tree_view_included_directories)); @@ -101,6 +102,7 @@ pub fn connect_button_search( let allowed_extensions = entry_allowed_extensions.get_text().as_str().to_string(); let hide_hard_links = check_button_settings_hide_hard_links.get_active(); let use_cache = check_button_settings_use_cache.get_active(); + let minimal_cache_file_size = entry_settings_cache_file_minimal_size.get_text().as_str().parse::().unwrap_or(2 * 1024 * 1024); let show_dialog = Arc::new(AtomicBool::new(true)); @@ -166,6 +168,7 @@ pub fn connect_button_search( df.set_excluded_items(excluded_items); df.set_allowed_extensions(allowed_extensions); df.set_minimal_file_size(minimal_file_size); + df.set_minimal_cache_file_size(minimal_cache_file_size); df.set_check_method(check_method); df.set_hash_type(hash_type); df.set_ignore_hard_links(hide_hard_links); diff --git a/czkawka_gui/src/gui_settings.rs b/czkawka_gui/src/gui_settings.rs index e1ca94d..e6218b4 100644 --- a/czkawka_gui/src/gui_settings.rs +++ b/czkawka_gui/src/gui_settings.rs @@ -15,6 +15,7 @@ pub struct GuiSettings { // Duplicates pub check_button_settings_hide_hard_links: gtk::CheckButton, + pub entry_settings_cache_file_minimal_size: gtk::Entry, // Similar Images pub check_button_settings_show_preview_similar_images: gtk::CheckButton, @@ -40,6 +41,7 @@ impl GuiSettings { // Duplicates let check_button_settings_hide_hard_links: gtk::CheckButton = builder.get_object("check_button_settings_hide_hard_links").unwrap(); + let entry_settings_cache_file_minimal_size: gtk::Entry = builder.get_object("entry_settings_cache_file_minimal_size").unwrap(); // Similar Images let check_button_settings_show_preview_similar_images: gtk::CheckButton = builder.get_object("check_button_settings_show_preview_similar_images").unwrap(); @@ -59,6 +61,7 @@ impl GuiSettings { check_button_settings_use_cache, check_button_settings_use_trash, check_button_settings_hide_hard_links, + entry_settings_cache_file_minimal_size, check_button_settings_show_preview_similar_images, button_settings_save_configuration, button_settings_load_configuration, diff --git a/czkawka_gui/src/saving_loading.rs b/czkawka_gui/src/saving_loading.rs index 227e149..135b1b2 100644 --- a/czkawka_gui/src/saving_loading.rs +++ b/czkawka_gui/src/saving_loading.rs @@ -130,6 +130,11 @@ pub fn save_configuration(gui_data: &GuiData, manual_execution: bool) { data_to_save.push("--use_trash:".to_string()); let check_button_settings_use_trash = gui_data.settings.check_button_settings_use_trash.clone(); data_to_save.push(check_button_settings_use_trash.get_active().to_string()); + + //// minimal cache file size + data_to_save.push("--cache_minimal_file_size:".to_string()); + let entry_settings_cache_file_minimal_size = gui_data.settings.entry_settings_cache_file_minimal_size.clone(); + data_to_save.push(entry_settings_cache_file_minimal_size.get_text().as_str().parse::().unwrap_or(2 * 1024 * 1024).to_string()); } // Creating/Opening config file @@ -181,6 +186,7 @@ enum TypeOfLoadedData { HideHardLinks, UseCache, UseTrash, + CacheMinimalSize, } pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) { @@ -227,6 +233,7 @@ pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) { let mut hide_hard_links: bool = true; let mut use_cache: bool = true; let mut use_trash: bool = false; + let mut cache_minimal_size: u64 = 2 * 1024 * 1024; let mut current_type = TypeOfLoadedData::None; for (line_number, line) in loaded_data.replace("\r\n", "\n").split('\n').enumerate() { @@ -260,6 +267,8 @@ pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) { current_type = TypeOfLoadedData::UseCache; } else if line.starts_with("--use_trash") { current_type = TypeOfLoadedData::UseTrash; + } else if line.starts_with("--cache_minimal_file_size") { + current_type = TypeOfLoadedData::CacheMinimalSize; } else if line.starts_with("--") { current_type = TypeOfLoadedData::None; add_text_to_text_view( @@ -403,6 +412,16 @@ pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) { ); } } + TypeOfLoadedData::CacheMinimalSize => { + if let Ok(number) = line.parse::() { + cache_minimal_size = number; + } else { + add_text_to_text_view( + &text_view_errors, + format!("Found invalid data in line {} \"{}\" isn't proper value(u64) when loading file {:?}", line_number, line, config_file).as_str(), + ); + } + } } } } @@ -457,6 +476,7 @@ pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) { gui_data.settings.check_button_settings_hide_hard_links.set_active(hide_hard_links); gui_data.settings.check_button_settings_use_cache.set_active(use_cache); gui_data.settings.check_button_settings_use_trash.set_active(use_trash); + gui_data.settings.entry_settings_cache_file_minimal_size.set_text(cache_minimal_size.to_string().as_str()); } else { gui_data.settings.check_button_settings_load_at_start.set_active(false); } @@ -540,6 +560,7 @@ pub fn reset_configuration(gui_data: &GuiData, manual_clearing: bool) { gui_data.settings.check_button_settings_hide_hard_links.set_active(true); gui_data.settings.check_button_settings_use_cache.set_active(true); gui_data.settings.check_button_settings_use_trash.set_active(false); + gui_data.settings.entry_settings_cache_file_minimal_size.set_text("2097152"); } if manual_clearing { add_text_to_text_view(&text_view_errors, "Current configuration was cleared.");