Allow to set minimal file size to caching (#321)

This commit is contained in:
Rafał Mikrut 2021-05-06 21:54:44 +02:00 committed by GitHub
parent 8e7ac4a2d7
commit b08cedd5a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 120 additions and 42 deletions

76
Cargo.lock generated
View File

@ -127,9 +127,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "base64"
version = "0.11.0"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bindgen"
@ -433,9 +433,9 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.3"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
checksum = "52fb27eab85b17fbb9f6fd667089e07d6a2eb8743d02639ee7f6a7a7729c9c94"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils",
@ -446,9 +446,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278"
dependencies = [
"autocfg",
"cfg-if 1.0.0",
@ -609,14 +609,14 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "flate2"
version = "1.0.14"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42"
checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0"
dependencies = [
"cfg-if 0.1.10",
"cfg-if 1.0.0",
"crc32fast",
"libc",
"miniz_oxide 0.3.7",
"miniz_oxide 0.4.4",
]
[[package]]
@ -1035,9 +1035,9 @@ dependencies = [
[[package]]
name = "img_hash"
version = "3.1.1"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52057a4b155b3cc36b9cd5b20faaa60f506e39727efcffad7d090f0163fe0c6d"
checksum = "5ea4eac6fc4f64ed363d5c210732b747bfa5ddd8a25ac347d887f298c3a70b49"
dependencies = [
"base64",
"image",
@ -1086,9 +1086,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
[[package]]
name = "jobserver"
version = "0.1.21"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2"
checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd"
dependencies = [
"libc",
]
@ -1136,9 +1136,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.93"
version = "0.2.94"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41"
checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"
[[package]]
name = "libloading"
@ -1152,9 +1152,9 @@ dependencies = [
[[package]]
name = "lock_api"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176"
checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb"
dependencies = [
"scopeguard",
]
@ -1179,9 +1179,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.3.4"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
[[package]]
name = "memoffset"
@ -1459,9 +1459,9 @@ checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
[[package]]
name = "open"
version = "1.6.0"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7e9f1bdf15cd1f5a00cc9002a733a6ee6d0ff562491852d59652471c4a389f7"
checksum = "1711eb4b31ce4ad35b0f316d8dfba4fe5c7ad601c448446d84aae7a896627b20"
dependencies = [
"which",
"winapi",
@ -1691,9 +1691,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.2.5"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9"
checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
dependencies = [
"bitflags",
]
@ -1710,18 +1710,18 @@ dependencies = [
[[package]]
name = "regex"
version = "1.4.5"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b"
dependencies = [
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.6.23"
version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "remove_dir_all"
@ -1822,9 +1822,9 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
[[package]]
name = "slab"
version = "0.4.2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
[[package]]
name = "slice-deque"
@ -1917,9 +1917,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
[[package]]
name = "syn"
version = "1.0.69"
version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb"
checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
dependencies = [
"proc-macro2",
"quote",
@ -2074,9 +2074,9 @@ checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
[[package]]
name = "unicode-xid"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "vec_map"
@ -2179,9 +2179,9 @@ dependencies = [
[[package]]
name = "weezl"
version = "0.1.4"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a32b378380f4e9869b22f0b5177c68a5519f03b3454fde0b291455ddbae266c"
checksum = "d8b77fdfd5a253be4ab714e4ffa3c49caf146b4de743e97510c0656cf90f1e8e"
[[package]]
name = "which"
@ -2232,9 +2232,9 @@ checksum = "e575e15bedf6e57b5c2d763ffc6c3c760143466cbd09d762d539680ab5992ded"
[[package]]
name = "zip"
version = "0.5.11"
version = "0.5.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8264fcea9b7a036a4a5103d7153e988dbc2ebbafb34f68a3c2d404b6b82d74b6"
checksum = "9c83dc9b784d252127720168abd71ea82bf8c3d96b17dc565b5e2a02854f2b27"
dependencies = [
"byteorder",
"bzip2",

View File

@ -17,9 +17,11 @@ pub enum Commands {
excluded_items: ExcludedItems,
#[structopt(short, long, parse(try_from_str = parse_minimal_file_size), default_value = "1024", help = "Minimum size in bytes", long_help = "Minimum size of checked files in bytes, assigning bigger value may speed up searching")]
minimal_file_size: u64,
#[structopt(short = "c", long, parse(try_from_str = parse_minimal_file_size), default_value = "2097152", help = "Minimum cached file size in bytes", long_help = "Minimum size of cached files in bytes, assigning bigger value may speed up will cause that lower amount of files will be cached, but loading of cache will be faster")]
minimal_cached_file_size: u64,
#[structopt(flatten)]
allowed_extensions: AllowedExtensions,
#[structopt(short, long, default_value = "HASH", parse(try_from_str = parse_checking_method), help = "Search method (NAME, SIZE, HASH, HASHMB)", long_help = "Methods to search files.\nNAME - Fast but but rarely usable,\nSIZE - Fast but not accurate, checking by the file's size,\nHASHMB - More accurate but slower, checking by the hash of the file's first mebibyte or\nHASH - The slowest method, checking by the hash of the entire file")]
#[structopt(short, long, default_value = "HASH", parse(try_from_str = parse_checking_method), help = "Search method (NAME, SIZE, HASH, HASHMB)", long_help = "Methods to search files.\nNAME - Fast but but rarely usable,\nSIZE - Fast but not accurate, checking by the file's size,\nHASHMB - More accurate but slower, checking by the hash of the file's first mebibyte\nHASH - The slowest method, checking by the hash of the entire file")]
search_method: CheckingMethod,
#[structopt(short = "D", long, default_value = "NONE", parse(try_from_str = parse_delete_method), help = "Delete method (AEN, AEO, ON, OO, HARD)", long_help = "Methods to delete the files.\nAEN - All files except the newest,\nAEO - All files except the oldest,\nON - Only 1 file, the newest,\nOO - Only 1 file, the oldest\nHARD - create hard link\nNONE - not delete files")]
delete_method: DeleteMethod,

View File

@ -33,6 +33,7 @@ fn main() {
excluded_directories,
excluded_items,
minimal_file_size,
minimal_cached_file_size,
allowed_extensions,
search_method,
delete_method,
@ -48,6 +49,7 @@ fn main() {
df.set_excluded_directory(excluded_directories.excluded_directories);
df.set_excluded_items(excluded_items.excluded_items);
df.set_minimal_file_size(minimal_file_size);
df.set_minimal_cache_file_size(minimal_cached_file_size);
df.set_allowed_extensions(allowed_extensions.allowed_extensions.join(","));
df.set_check_method(search_method);
df.set_delete_method(delete_method);

View File

@ -152,6 +152,7 @@ pub struct DuplicateFinder {
dryrun: bool,
stopped_search: bool,
use_cache: bool,
minimal_cache_file_size: u64,
}
impl DuplicateFinder {
@ -174,6 +175,7 @@ impl DuplicateFinder {
hash_type: HashType::Blake3,
dryrun: false,
use_cache: true,
minimal_cache_file_size: 2 * 1024 * 1024, // By default cache only >= 1MB files
}
}
@ -219,6 +221,10 @@ impl DuplicateFinder {
self.stopped_search
}
pub fn set_minimal_cache_file_size(&mut self, minimal_cache_file_size: u64) {
self.minimal_cache_file_size = minimal_cache_file_size;
}
pub const fn get_files_sorted_by_names(&self) -> &BTreeMap<String, Vec<FileEntry>> {
&self.files_with_identical_names
}
@ -909,7 +915,7 @@ impl DuplicateFinder {
}
}
}
save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type);
save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type, self.minimal_cache_file_size);
}
}
_ => panic!("What"),
@ -1328,7 +1334,7 @@ pub fn make_hard_link(src: &Path, dst: &Path) -> io::Result<()> {
result
}
fn save_hashes_to_file(hashmap: &HashMap<String, FileEntry>, text_messages: &mut Messages, type_of_hash: &HashType) {
fn save_hashes_to_file(hashmap: &HashMap<String, FileEntry>, text_messages: &mut Messages, type_of_hash: &HashType, minimal_cache_file_size: u64) {
if let Some(proj_dirs) = ProjectDirs::from("pl", "Qarmin", "Czkawka") {
let cache_dir = PathBuf::from(proj_dirs.cache_dir());
if cache_dir.exists() {
@ -1352,7 +1358,7 @@ fn save_hashes_to_file(hashmap: &HashMap<String, FileEntry>, text_messages: &mut
for file_entry in hashmap.values() {
// Only cache bigger than 5MB files
if file_entry.size > 5 * 1024 * 1024 {
if file_entry.size >= minimal_cache_file_size {
let string: String = format!("{}//{}//{}//{}", file_entry.path.display(), file_entry.size, file_entry.modified_date, file_entry.hash);
if writeln!(writer, "{}", string).is_err() {

View File

@ -2614,6 +2614,47 @@ This program is free to use and will always be.
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkBox">
<property name="visible">True</property>
<property name="can-focus">False</property>
<property name="margin-start">4</property>
<property name="margin-end">4</property>
<child>
<object class="GtkLabel">
<property name="visible">True</property>
<property name="can-focus">False</property>
<property name="label" translatable="yes">Minimal cached file size in bytes</property>
</object>
<packing>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkEntry" id="entry_settings_cache_file_minimal_size">
<property name="visible">True</property>
<property name="can-focus">True</property>
<property name="max-length">15</property>
<property name="text" translatable="yes">2097152</property>
<property name="caps-lock-warning">False</property>
<property name="input-purpose">number</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">False</property>
<property name="pack-type">end</property>
<property name="position">1</property>
</packing>
</child>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">2</property>
</packing>
</child>
</object>
<packing>
<property name="position">1</property>

View File

@ -92,6 +92,7 @@ pub fn connect_button_search(
let radio_button_hash_type_xxh3 = gui_data.main_notebook.radio_button_hash_type_xxh3.clone();
let check_button_settings_hide_hard_links = gui_data.settings.check_button_settings_hide_hard_links.clone();
let check_button_settings_use_cache = gui_data.settings.check_button_settings_use_cache.clone();
let entry_settings_cache_file_minimal_size = gui_data.settings.entry_settings_cache_file_minimal_size.clone();
buttons_search_clone.connect_clicked(move |_| {
let included_directories = get_path_buf_from_vector_of_strings(get_string_from_list_store(&tree_view_included_directories));
@ -101,6 +102,7 @@ pub fn connect_button_search(
let allowed_extensions = entry_allowed_extensions.get_text().as_str().to_string();
let hide_hard_links = check_button_settings_hide_hard_links.get_active();
let use_cache = check_button_settings_use_cache.get_active();
let minimal_cache_file_size = entry_settings_cache_file_minimal_size.get_text().as_str().parse::<u64>().unwrap_or(2 * 1024 * 1024);
let show_dialog = Arc::new(AtomicBool::new(true));
@ -166,6 +168,7 @@ pub fn connect_button_search(
df.set_excluded_items(excluded_items);
df.set_allowed_extensions(allowed_extensions);
df.set_minimal_file_size(minimal_file_size);
df.set_minimal_cache_file_size(minimal_cache_file_size);
df.set_check_method(check_method);
df.set_hash_type(hash_type);
df.set_ignore_hard_links(hide_hard_links);

View File

@ -15,6 +15,7 @@ pub struct GuiSettings {
// Duplicates
pub check_button_settings_hide_hard_links: gtk::CheckButton,
pub entry_settings_cache_file_minimal_size: gtk::Entry,
// Similar Images
pub check_button_settings_show_preview_similar_images: gtk::CheckButton,
@ -40,6 +41,7 @@ impl GuiSettings {
// Duplicates
let check_button_settings_hide_hard_links: gtk::CheckButton = builder.get_object("check_button_settings_hide_hard_links").unwrap();
let entry_settings_cache_file_minimal_size: gtk::Entry = builder.get_object("entry_settings_cache_file_minimal_size").unwrap();
// Similar Images
let check_button_settings_show_preview_similar_images: gtk::CheckButton = builder.get_object("check_button_settings_show_preview_similar_images").unwrap();
@ -59,6 +61,7 @@ impl GuiSettings {
check_button_settings_use_cache,
check_button_settings_use_trash,
check_button_settings_hide_hard_links,
entry_settings_cache_file_minimal_size,
check_button_settings_show_preview_similar_images,
button_settings_save_configuration,
button_settings_load_configuration,

View File

@ -130,6 +130,11 @@ pub fn save_configuration(gui_data: &GuiData, manual_execution: bool) {
data_to_save.push("--use_trash:".to_string());
let check_button_settings_use_trash = gui_data.settings.check_button_settings_use_trash.clone();
data_to_save.push(check_button_settings_use_trash.get_active().to_string());
//// minimal cache file size
data_to_save.push("--cache_minimal_file_size:".to_string());
let entry_settings_cache_file_minimal_size = gui_data.settings.entry_settings_cache_file_minimal_size.clone();
data_to_save.push(entry_settings_cache_file_minimal_size.get_text().as_str().parse::<u64>().unwrap_or(2 * 1024 * 1024).to_string());
}
// Creating/Opening config file
@ -181,6 +186,7 @@ enum TypeOfLoadedData {
HideHardLinks,
UseCache,
UseTrash,
CacheMinimalSize,
}
pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) {
@ -227,6 +233,7 @@ pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) {
let mut hide_hard_links: bool = true;
let mut use_cache: bool = true;
let mut use_trash: bool = false;
let mut cache_minimal_size: u64 = 2 * 1024 * 1024;
let mut current_type = TypeOfLoadedData::None;
for (line_number, line) in loaded_data.replace("\r\n", "\n").split('\n').enumerate() {
@ -260,6 +267,8 @@ pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) {
current_type = TypeOfLoadedData::UseCache;
} else if line.starts_with("--use_trash") {
current_type = TypeOfLoadedData::UseTrash;
} else if line.starts_with("--cache_minimal_file_size") {
current_type = TypeOfLoadedData::CacheMinimalSize;
} else if line.starts_with("--") {
current_type = TypeOfLoadedData::None;
add_text_to_text_view(
@ -403,6 +412,16 @@ pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) {
);
}
}
TypeOfLoadedData::CacheMinimalSize => {
if let Ok(number) = line.parse::<u64>() {
cache_minimal_size = number;
} else {
add_text_to_text_view(
&text_view_errors,
format!("Found invalid data in line {} \"{}\" isn't proper value(u64) when loading file {:?}", line_number, line, config_file).as_str(),
);
}
}
}
}
}
@ -457,6 +476,7 @@ pub fn load_configuration(gui_data: &GuiData, manual_execution: bool) {
gui_data.settings.check_button_settings_hide_hard_links.set_active(hide_hard_links);
gui_data.settings.check_button_settings_use_cache.set_active(use_cache);
gui_data.settings.check_button_settings_use_trash.set_active(use_trash);
gui_data.settings.entry_settings_cache_file_minimal_size.set_text(cache_minimal_size.to_string().as_str());
} else {
gui_data.settings.check_button_settings_load_at_start.set_active(false);
}
@ -540,6 +560,7 @@ pub fn reset_configuration(gui_data: &GuiData, manual_clearing: bool) {
gui_data.settings.check_button_settings_hide_hard_links.set_active(true);
gui_data.settings.check_button_settings_use_cache.set_active(true);
gui_data.settings.check_button_settings_use_trash.set_active(false);
gui_data.settings.entry_settings_cache_file_minimal_size.set_text("2097152");
}
if manual_clearing {
add_text_to_text_view(&text_view_errors, "Current configuration was cleared.");