diff --git a/Cargo.lock b/Cargo.lock index 93610ac..1912520 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,16 +21,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", - "cipher", + "cipher 0.3.0", "cpufeatures", "opaque-debug", ] [[package]] -name = "aho-corasick" -version = "0.7.20" +name = "aes" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures", +] + +[[package]] +name = "aho-corasick" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -46,49 +57,58 @@ dependencies = [ [[package]] name = "anstream" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", + "anstyle-query", "anstyle-wincon", - "concolor-override", - "concolor-query", + "colorchoice", "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "0.3.5" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" [[package]] name = "anstyle-parse" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" dependencies = [ "utf8parse", ] [[package]] -name = "anstyle-wincon" -version = "0.2.0" +name = "anstyle-query" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "arc-swap" @@ -116,7 +136,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -175,9 +195,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" +checksum = "24a6904aef64d73cf10ab17ebace7befb918b82164785cb89907993be7f83813" [[package]] name = "bk-tree" @@ -219,26 +239,19 @@ dependencies = [ ] [[package]] -name = "block-modes" -version = "0.8.1" +name = "block-padding" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "block-padding", - "cipher", + "generic-array", ] -[[package]] -name = "block-padding" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" - [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" [[package]] name = "bytemuck" @@ -298,6 +311,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "cc" version = "1.0.79" @@ -317,11 +339,12 @@ dependencies = [ [[package]] name = "cfg-expr" -version = "0.14.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35b255461940a32985c627ce82900867c61db1659764d3675ea81963f72a4c6" +checksum = "c8790cf1286da485c72cf5fc7aeba308438800036ec67d89425924c4807268c9" dependencies = [ "smallvec", + "target-lexicon", ] [[package]] @@ -355,10 +378,20 @@ dependencies = [ ] [[package]] -name = "clap" -version = "4.2.1" +name = "cipher" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d70680e56dc65cb226c361aaa4e4a16d1f7e082bfed9ffceaee39c2012384ec" dependencies = [ "clap_builder", "clap_derive", @@ -367,9 +400,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.2.1" +version = "4.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +checksum = "3fad499d5e07338414687350c5fdb82b1ab0001e9b26aa6275deccb684b14164" dependencies = [ "anstream", "anstyle", @@ -387,7 +420,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -413,19 +446,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] -name = "concolor-override" +name = "colorchoice" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" - -[[package]] -name = "concolor-query" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" -dependencies = [ - "windows-sys 0.45.0", -] +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "constant_time_eq" @@ -447,9 +471,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] @@ -465,9 +489,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -546,7 +570,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -563,7 +587,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -582,7 +606,7 @@ dependencies = [ "anyhow", "audio_checker", "bincode", - "bitflags 2.0.2", + "bitflags 2.2.1", "bk-tree", "blake3", "crc32fast", @@ -607,9 +631,11 @@ dependencies = [ "rawloader", "rayon", "rust-embed", + "rusty-chromaprint", "serde", "serde_json", "state", + "symphonia", "tempfile", "vid_dup_finder_lib", "xxhash-rust", @@ -756,18 +782,18 @@ checksum = "48016319042fb7c87b78d2993084a831793a897a5cd1a2a67cab9d1eeb4b7d76" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] name = "errno" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -790,7 +816,7 @@ dependencies = [ "flume", "half", "lebe", - "miniz_oxide", + "miniz_oxide 0.6.2", "rayon-core", "smallvec", "zune-inflate", @@ -825,6 +851,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "fdeflate" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d329bdeac514ee06249dabc27877490f17f5d371ec693360768b838e19f3ae10" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ffmpeg_cmdline_utils" version = "0.1.2" @@ -860,12 +895,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.7.1", ] [[package]] @@ -1008,7 +1043,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -1103,15 +1138,15 @@ dependencies = [ [[package]] name = "generator" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33a20a288a94683f5f4da0adecdbe095c94a77c295e514cc6484e9394dd8376e" +checksum = "f3e123d9ae7c02966b4d892e550bdc32164f05853cd40ab570650ad600596a8a" dependencies = [ "cc", "libc", "log", "rustversion", - "windows 0.44.0", + "windows 0.48.0", ] [[package]] @@ -1126,9 +1161,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "js-sys", @@ -1149,9 +1184,9 @@ dependencies = [ [[package]] name = "gio" -version = "0.17.4" +version = "0.17.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261a3b4e922ec676d1c27ac466218c38cf5dcb49a759129e54bb5046e442125" +checksum = "d14522e56c6bcb6f7a3aebc25cbcfb06776af4c0c25232b601b4383252d7cb92" dependencies = [ "bitflags 1.3.2", "futures-channel", @@ -1182,9 +1217,9 @@ dependencies = [ [[package]] name = "glib" -version = "0.17.5" +version = "0.17.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb53061756195d76969292c2d2e329e01259276524a9bae6c9b73af62854773" +checksum = "a7f1de7cbde31ea4f0a919453a2dcece5d54d5b70e08f8ad254dc4840f5f09b6" dependencies = [ "bitflags 1.3.2", "futures-channel", @@ -1205,9 +1240,9 @@ dependencies = [ [[package]] name = "glib-macros" -version = "0.17.7" +version = "0.17.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc4cf346122086f196260783aa58987190dbd5f43bfab01946d2bf9786e8d9ef" +checksum = "0a7206c5c03851ef126ea1444990e81fdd6765fb799d5bc694e4897ca01bb97f" dependencies = [ "anyhow", "heck", @@ -1311,9 +1346,9 @@ dependencies = [ [[package]] name = "gtk4" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e30e124b5a605f6f5513db13958bfcd51d746607b20bc7bb718b33e303274ed" +checksum = "b28a32a04cd75cef14a0983f8b0c669e0fe152a0a7725accdeb594e2c764c88b" dependencies = [ "bitflags 1.3.2", "cairo-rs", @@ -1334,9 +1369,9 @@ dependencies = [ [[package]] name = "gtk4-macros" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f041a797fb098bfb06e432c61738133604bfa3af57f13f1da3b9d46271422ef0" +checksum = "6a4d6b61570f76d3ee542d984da443b1cd69b6105264c61afec3abed08c2500f" dependencies = [ "anyhow", "proc-macro-crate", @@ -1609,6 +1644,16 @@ dependencies = [ "adler32", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -1649,15 +1694,34 @@ dependencies = [ ] [[package]] -name = "is-terminal" -version = "0.4.6" +name = "is-docker" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +dependencies = [ + "once_cell", +] + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys 0.45.0", + "windows-sys 0.48.0", +] + +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", ] [[package]] @@ -1716,9 +1780,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" [[package]] name = "libheif-rs" @@ -1763,9 +1827,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c" [[package]] name = "locale_config" @@ -1792,9 +1856,9 @@ dependencies = [ [[package]] name = "lofty" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8d7482b0444750bb69ce3abaae2178e68cf8bbd85019ae7f19c404b816c7bd" +checksum = "fd1b8e18439c8fabf316e0a87e9cdca9667e90bcf5a080946a264fd60bbed5e8" dependencies = [ "base64 0.21.0", "byteorder", @@ -1905,6 +1969,16 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", + "simd-adler32", +] + [[package]] name = "multicache" version = "0.6.1" @@ -2043,10 +2117,11 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "open" -version = "4.0.1" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "075c5203b3a2b698bc72c6c10b1f6263182135751d5013ea66e8a4b3d0562a43" +checksum = "d16814a067484415fda653868c9be0ac5f2abd2ef5d951082a5f2fe1b3662944" dependencies = [ + "is-wsl", "pathdiff", ] @@ -2142,13 +2217,13 @@ dependencies = [ [[package]] name = "pdf" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbf9b11d32e9323b219368cc9a858485f3408901721a28b1b7b1aa18a747d69" +checksum = "e375ec076445f61d4dbc4636e9e788f841d279c65d6fea8a3875caddd4f2dd82" dependencies = [ - "aes", + "aes 0.8.2", "bitflags 1.3.2", - "block-modes", + "cbc", "datasize", "deflate", "fax", @@ -2224,14 +2299,15 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "png" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d708eaf860a19b19ce538740d2b4bdeeb8337fa53f7738455e706623ad5c638" +checksum = "aaeebc51f9e7d2c150d3f3bfeb667f2aa985db5ef1e3d212847bdedb488beeaa" dependencies = [ "bitflags 1.3.2", "crc32fast", + "fdeflate", "flate2", - "miniz_oxide", + "miniz_oxide 0.7.1", ] [[package]] @@ -2377,6 +2453,15 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "realfft" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6b8e8f0c6d2234aa58048d7290c60bf92cd36fd2888cd8331c66ad4f2e1d2" +dependencies = [ + "rustfft 6.1.0", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2408,13 +2493,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.1", ] [[package]] @@ -2423,7 +2508,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -2432,6 +2517,24 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + +[[package]] +name = "rubato" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd70209c27d5b08f5528bdc779ea3ffb418954e28987f9f9775c6eac41003f9c" +dependencies = [ + "num-complex 0.4.3", + "num-integer", + "num-traits", + "realfft", +] + [[package]] name = "rust-embed" version = "6.6.1" @@ -2530,16 +2633,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.7" +version = "0.37.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aae838e49b3d63e9274e1c01833cc8139d3fec468c3b84688c628f44b1ae11d" +checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2548,6 +2651,16 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +[[package]] +name = "rusty-chromaprint" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a224821c3208db13134f398c2d92ed81267ef4f65ac8dff670c00b829faac" +dependencies = [ + "rubato", + "rustfft 6.1.0", +] + [[package]] name = "ryu" version = "1.0.13" @@ -2595,29 +2708,29 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] name = "serde_json" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -2961,9 +3074,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.13" +version = "2.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" dependencies = [ "proc-macro2", "quote", @@ -2972,9 +3085,9 @@ dependencies = [ [[package]] name = "system-deps" -version = "6.0.4" +version = "6.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555fc8147af6256f3931a36bb83ad0023240ce9cf2b319dec8236fd1f220b05f" +checksum = "d0fe581ad25d11420b873cf9aedaca0419c2b411487b134d4d21065f3d092055" dependencies = [ "cfg-expr", "heck", @@ -2983,6 +3096,12 @@ dependencies = [ "version-compare", ] +[[package]] +name = "target-lexicon" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" + [[package]] name = "tempfile" version = "3.5.0" @@ -3022,7 +3141,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -3154,13 +3273,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] @@ -3186,9 +3305,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", "nu-ansi-term", @@ -3323,9 +3442,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.3.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" +checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" [[package]] name = "valuable" @@ -3627,9 +3746,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.1" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" dependencies = [ "memchr", ] @@ -3655,7 +3774,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" dependencies = [ - "aes", + "aes 0.7.5", "byteorder", "bzip2", "constant_time_eq 0.1.5", @@ -3670,9 +3789,9 @@ dependencies = [ [[package]] name = "zune-inflate" -version = "0.2.53" +version = "0.2.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "440a08fd59c6442e4b846ea9b10386c38307eae728b216e1ab2c305d1c9daaf8" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" dependencies = [ "simd-adler32", ] diff --git a/czkawka_cli/Cargo.toml b/czkawka_cli/Cargo.toml index 2f806c1..35dfa9b 100644 --- a/czkawka_cli/Cargo.toml +++ b/czkawka_cli/Cargo.toml @@ -13,7 +13,7 @@ repository = "https://github.com/qarmin/czkawka" clap = { version = "4.2", features = ["derive"] } # For enum types -image_hasher = "1.1.2" +image_hasher = "1.1" [dependencies.czkawka_core] path = "../czkawka_core" diff --git a/czkawka_cli/src/main.rs b/czkawka_cli/src/main.rs index a98ba6d..8f81e92 100644 --- a/czkawka_cli/src/main.rs +++ b/czkawka_cli/src/main.rs @@ -4,10 +4,6 @@ use std::process; use clap::Parser; -use crate::commands::{ - Args, BadExtensionsArgs, BiggestFilesArgs, BrokenFilesArgs, DuplicatesArgs, EmptyFilesArgs, EmptyFoldersArgs, InvalidSymlinksArgs, SameMusicArgs, SimilarImagesArgs, - SimilarVideosArgs, TemporaryArgs, -}; use commands::Commands; use czkawka_core::big_file::SearchMode; use czkawka_core::common::{get_number_of_threads, set_default_number_of_threads}; @@ -28,6 +24,11 @@ use czkawka_core::{ temporary::{self, Temporary}, }; +use crate::commands::{ + Args, BadExtensionsArgs, BiggestFilesArgs, BrokenFilesArgs, DuplicatesArgs, EmptyFilesArgs, EmptyFoldersArgs, InvalidSymlinksArgs, SameMusicArgs, SimilarImagesArgs, + SimilarVideosArgs, TemporaryArgs, +}; + mod commands; fn main() { diff --git a/czkawka_core/Cargo.toml b/czkawka_core/Cargo.toml index 165eb79..db4ac27 100644 --- a/czkawka_core/Cargo.toml +++ b/czkawka_core/Cargo.toml @@ -11,68 +11,72 @@ repository = "https://github.com/qarmin/czkawka" [dependencies] -humansize = "2.1.3" -rayon = "1.7.0" -crossbeam-channel = "0.5.7" +humansize = "2.1" +rayon = "1.7" +crossbeam-channel = "0.5" # For saving/loading config files to specific directories -directories-next = "2.0.0" +directories-next = "2.0" # Needed by similar images -image_hasher = "1.1.2" -bk-tree = "0.5.0" -image = "0.24.6" -hamming = "0.1.3" +image_hasher = "1.1" +bk-tree = "0.5" +image = "0.24" +hamming = "0.1" # Needed by same music -bitflags = "2.0.2" -lofty = "0.12.0" +bitflags = "2.2" +lofty = "0.12" # Futures - needed by async progress sender futures = "0.3.28" # Needed by broken files -zip = { version = "0.6.4", features = ["aes-crypto", "bzip2", "deflate", "time"], default-features = false } -audio_checker = "0.1.0" -pdf = "0.8.0" +zip = { version = "0.6", features = ["aes-crypto", "bzip2", "deflate", "time"], default-features = false } +audio_checker = "0.1" +pdf = "0.8" + +# Needed by audio similarity feature +rusty-chromaprint = "0.1" +symphonia = { version = "0.5", features = ["mp3", "aac", "alac", "flac", "isomp4", "mkv", "ogg", "pcm", "vorbis", "wav"] } # Hashes for duplicate files -blake3 = "1.3.3" -crc32fast = "1.3.2" -xxhash-rust = { version = "0.8.6", features = ["xxh3"] } +blake3 = "1.3" +crc32fast = "1.3" +xxhash-rust = { version = "0.8", features = ["xxh3"] } -tempfile = "3.5.0" +tempfile = "3.5" # Video Duplicates -vid_dup_finder_lib = "0.1.1" -ffmpeg_cmdline_utils = "0.1.2" +vid_dup_finder_lib = "0.1" +ffmpeg_cmdline_utils = "0.1" # Saving/Loading Cache serde = "1.0" -bincode = "1.3.3" +bincode = "1.3" serde_json = "1.0" # Language -i18n-embed = { version = "0.13.8", features = ["fluent-system", "desktop-requester"] } -i18n-embed-fl = "0.6.6" -rust-embed = "6.6.1" -once_cell = "1.17.1" +i18n-embed = { version = "0.13", features = ["fluent-system", "desktop-requester"] } +i18n-embed-fl = "0.6" +rust-embed = "6.6" +once_cell = "1.17" # Raw image files -rawloader = "0.37.1" -imagepipe = "0.5.0" +rawloader = "0.37" +imagepipe = "0.5" # Checking for invalid extensions -mime_guess = "2.0.4" -infer = "0.13.0" +mime_guess = "2.0" +infer = "0.13" -num_cpus = "1.15.0" +num_cpus = "1.15" # Heif/Heic libheif-rs = { version = "0.18.0", optional = true } # Do not upgrade now, since Ubuntu 22.04 not works with newer version anyhow = { version = "1.0", optional = true } -state = "0.5.3" +state = "0.5" [features] default = [] diff --git a/czkawka_core/src/bad_extensions.rs b/czkawka_core/src/bad_extensions.rs index 5f2259a..f90739d 100644 --- a/czkawka_core/src/bad_extensions.rs +++ b/czkawka_core/src/bad_extensions.rs @@ -2,18 +2,18 @@ use std::collections::{BTreeSet, HashMap}; use std::fs::File; use std::io::prelude::*; use std::io::BufWriter; +use std::mem; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::sleep; -use std::time::{Duration, SystemTime}; -use std::{mem, thread}; +use std::time::SystemTime; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use mime_guess::get_mime_extensions; use rayon::prelude::*; -use crate::common::{Common, LOOP_DURATION}; +use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; @@ -25,7 +25,7 @@ static DISABLED_EXTENSIONS: &[&str] = &["file", "cache", "bak", "data"]; // Such // This adds several workarounds for bugs/invalid recognizing types by external libraries // ("real_content_extension", "current_file_extension") -static WORKAROUNDS: &[(&str, &str)] = &[ +const WORKAROUNDS: &[(&str, &str)] = &[ // Wine/Windows ("der", "cat"), ("exe", "acm"), @@ -208,7 +208,7 @@ impl BadExtensions { } } - pub fn find_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.directories.optimize_directories(self.recursive_search, &mut self.text_messages); if !self.check_files(stop_receiver, progress_sender) { self.stopped_search = true; @@ -283,7 +283,7 @@ impl BadExtensions { self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages); } - fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let result = DirTraversalBuilder::new() .root_dirs(self.directories.included_directories.clone()) .group_by(|_fe| ()) @@ -317,59 +317,66 @@ impl BadExtensions { } } - fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let system_time = SystemTime::now(); - let include_files_without_extension = self.include_files_without_extension; - let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - let entries_to_check = self.files_to_check.len(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - checking_method: CheckingMethod::None, - current_stage: 1, - max_stage: 1, - entries_checked: atomic_file_counter.load(Ordering::Relaxed), - entries_to_check, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common( + progress_sender, + &progress_thread_run, + &atomic_counter, + 1, + 1, + self.files_to_check.len(), + CheckingMethod::None, + ); let mut files_to_check = Default::default(); mem::swap(&mut files_to_check, &mut self.files_to_check); - //// PROGRESS THREAD END let mut hashmap_workarounds: HashMap<&str, Vec<&str>> = Default::default(); for (proper, found) in WORKAROUNDS { - // This should be enabled when items will have only 1 possible workaround items + // This should be enabled when items will have only 1 possible workaround items, but looks that some have 2 or even more, so at least for now this is disabled // if hashmap_workarounds.contains_key(found) { // panic!("Already have {} key", found); // } hashmap_workarounds.entry(found).or_insert_with(Vec::new).push(proper); } - self.bad_extensions_files = files_to_check + self.bad_extensions_files = self.verify_extensions(files_to_check, &atomic_counter, stop_receiver, &check_was_stopped, &hashmap_workarounds); + + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); + + // Break if stop was clicked + if check_was_stopped.load(Ordering::Relaxed) { + return false; + } + + self.information.number_of_files_with_bad_extension = self.bad_extensions_files.len(); + + Common::print_time(system_time, SystemTime::now(), "bad extension finding"); + + // Clean unused data + self.files_to_check = Default::default(); + + true + } + + fn verify_extensions( + &self, + files_to_check: Vec, + atomic_counter: &Arc, + stop_receiver: Option<&Receiver<()>>, + check_was_stopped: &AtomicBool, + hashmap_workarounds: &HashMap<&str, Vec<&str>>, + ) -> Vec { + files_to_check .into_par_iter() .map(|file_entry| { - println!("{:?}", file_entry.path); - atomic_file_counter.fetch_add(1, Ordering::Relaxed); + atomic_counter.fetch_add(1, Ordering::Relaxed); if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { check_was_stopped.store(true, Ordering::Relaxed); return None; @@ -385,69 +392,21 @@ impl BadExtensions { }; let proper_extension = kind.extension(); - // Extract current extension from file - let current_extension; - if let Some(extension) = file_entry.path.extension() { - let extension = extension.to_string_lossy().to_lowercase(); - if DISABLED_EXTENSIONS.contains(&extension.as_str()) { - return Some(None); - } - // Text longer than 10 characters is not considered as extension - if extension.len() > 10 { - current_extension = String::new(); - } else { - current_extension = extension; - } - } else { - current_extension = String::new(); - } - - // Already have proper extension, no need to do more things - if current_extension == proper_extension { + let Some(current_extension) = self.get_and_validate_extension(&file_entry, proper_extension) else { return Some(None); - } + }; // Check for all extensions that file can use(not sure if it is worth to do it) - let mut all_available_extensions: BTreeSet<&str> = Default::default(); - let think_extension = if current_extension.is_empty() { - String::new() - } else { - for mim in mime_guess::from_ext(proper_extension) { - if let Some(all_ext) = get_mime_extensions(&mim) { - for ext in all_ext { - all_available_extensions.insert(ext); - } - } - } - - // Workarounds - if let Some(vec_pre) = hashmap_workarounds.get(current_extension.as_str()) { - for pre in vec_pre { - if all_available_extensions.contains(pre) { - all_available_extensions.insert(current_extension.as_str()); - break; - } - } - } - - let mut guessed_multiple_extensions = format!("({proper_extension}) - "); - for ext in &all_available_extensions { - guessed_multiple_extensions.push_str(ext); - guessed_multiple_extensions.push(','); - } - guessed_multiple_extensions.pop(); - - guessed_multiple_extensions - }; + let (mut all_available_extensions, valid_extensions) = self.check_for_all_extensions_that_file_can_use(hashmap_workarounds, ¤t_extension, proper_extension); if all_available_extensions.is_empty() { // Not found any extension return Some(None); } else if current_extension.is_empty() { - if !include_files_without_extension { + if !self.include_files_without_extension { return Some(None); } - } else if all_available_extensions.take(¤t_extension.as_str()).is_some() { + } else if all_available_extensions.take(¤t_extension).is_some() { // Found proper extension return Some(None); } @@ -457,31 +416,78 @@ impl BadExtensions { modified_date: file_entry.modified_date, size: file_entry.size, current_extension, - proper_extensions: think_extension, + proper_extensions: valid_extensions, })) }) .while_some() .filter(Option::is_some) .map(Option::unwrap) - .collect::>(); + .collect::>() + } - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - - // Break if stop was clicked - if check_was_stopped.load(Ordering::Relaxed) { - return false; + fn get_and_validate_extension(&self, file_entry: &FileEntry, proper_extension: &str) -> Option { + let current_extension; + // Extract current extension from file + if let Some(extension) = file_entry.path.extension() { + let extension = extension.to_string_lossy().to_lowercase(); + if DISABLED_EXTENSIONS.contains(&extension.as_str()) { + return None; + } + // Text longer than 10 characters is not considered as extension + if extension.len() > 10 { + current_extension = String::new(); + } else { + current_extension = extension; + } + } else { + current_extension = String::new(); } - self.information.number_of_files_with_bad_extension = self.bad_extensions_files.len(); + // Already have proper extension, no need to do more things + if current_extension == proper_extension { + return None; + } + Some(current_extension) + } - Common::print_time(system_time, SystemTime::now(), "bad extension finding"); + fn check_for_all_extensions_that_file_can_use( + &self, + hashmap_workarounds: &HashMap<&str, Vec<&str>>, + current_extension: &str, + proper_extension: &str, + ) -> (BTreeSet, String) { + let mut all_available_extensions: BTreeSet = Default::default(); + let valid_extensions = if current_extension.is_empty() { + String::new() + } else { + for mim in mime_guess::from_ext(proper_extension) { + if let Some(all_ext) = get_mime_extensions(&mim) { + for ext in all_ext { + all_available_extensions.insert((*ext).to_string()); + } + } + } - // Clean unused data - self.files_to_check = Default::default(); + // Workarounds + if let Some(vec_pre) = hashmap_workarounds.get(current_extension) { + for pre in vec_pre { + if all_available_extensions.contains(*pre) { + all_available_extensions.insert(current_extension.to_string()); + break; + } + } + } - true + let mut guessed_multiple_extensions = format!("({proper_extension}) - "); + for ext in &all_available_extensions { + guessed_multiple_extensions.push_str(ext); + guessed_multiple_extensions.push(','); + } + guessed_multiple_extensions.pop(); + + guessed_multiple_extensions + }; + (all_available_extensions, valid_extensions) } } diff --git a/czkawka_core/src/big_file.rs b/czkawka_core/src/big_file.rs index b4a8412..cbfc848 100644 --- a/czkawka_core/src/big_file.rs +++ b/czkawka_core/src/big_file.rs @@ -1,34 +1,27 @@ use std::collections::BTreeMap; -use std::fs::{File, Metadata}; +use std::fs; +use std::fs::{DirEntry, File, Metadata}; use std::io::{BufWriter, Write}; -use std::path::PathBuf; -use std::sync::atomic::Ordering; -use std::sync::atomic::{AtomicBool, AtomicU64}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::sleep; -use std::time::Duration; -use std::time::{SystemTime, UNIX_EPOCH}; -use std::{fs, thread}; +use std::time::SystemTime; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use humansize::format_size; use humansize::BINARY; use rayon::prelude::*; -use crate::common::split_path; -use crate::common::{Common, LOOP_DURATION}; +use crate::common::Common; +use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, split_path}; +use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; use crate::common_items::ExcludedItems; use crate::common_messages::Messages; use crate::common_traits::{DebugPrint, PrintResults, SaveResults}; -use crate::flc; -use crate::localizer_core::generate_translation_hashmap; - -#[derive(Debug)] -pub struct ProgressData { - pub files_checked: usize, -} #[derive(Clone)] pub struct FileEntry { @@ -95,7 +88,7 @@ impl BigFile { } } - pub fn find_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.optimize_directories(); if !self.look_for_big_files(stop_receiver, progress_sender) { self.stopped_search = true; @@ -148,7 +141,7 @@ impl BigFile { self.allowed_extensions.set_allowed_extensions(allowed_extensions, &mut self.text_messages); } - fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let start_time: SystemTime = SystemTime::now(); let mut folders_to_check: Vec = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector let mut old_map: BTreeMap> = Default::default(); @@ -158,36 +151,13 @@ impl BigFile { folders_to_check.push(id.clone()); } - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 0, 0, CheckingMethod::None); - let atomic_file_counter = Arc::new(AtomicU64::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - files_checked: atomic_file_counter.load(Ordering::Relaxed) as usize, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - - //// PROGRESS THREAD END while !folders_to_check.is_empty() { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); return false; } @@ -197,117 +167,29 @@ impl BigFile { let mut dir_result = vec![]; let mut warnings = vec![]; let mut fe_result = vec![]; - // Read current dir children - let read_dir = match fs::read_dir(current_folder) { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_open_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - return (dir_result, warnings, fe_result); - } + + let Some(read_dir) = common_read_dir(current_folder, &mut warnings) else { + return (dir_result, warnings, fe_result); }; // Check every sub folder/file/link etc. - 'dir: for entry in read_dir { - let entry_data = match entry { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_entry_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } - }; - let metadata: Metadata = match entry_data.metadata() { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_metadata_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } + for entry in read_dir { + let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else { + continue; }; + if metadata.is_dir() { - if !self.recursive_search { - continue 'dir; - } - - let next_folder = current_folder.join(entry_data.file_name()); - if self.directories.is_excluded(&next_folder) { - continue 'dir; - } - - if self.excluded_items.is_excluded(&next_folder) { - continue 'dir; - } - - #[cfg(target_family = "unix")] - if self.directories.exclude_other_filesystems() { - match self.directories.is_on_other_filesystems(&next_folder) { - Ok(true) => continue 'dir, - Err(e) => warnings.push(e.to_string()), - _ => (), - } - } - - dir_result.push(next_folder); + check_folder_children( + &mut dir_result, + &mut warnings, + current_folder, + entry_data, + self.recursive_search, + &self.directories, + &self.excluded_items, + ); } else if metadata.is_file() { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - - if metadata.len() == 0 { - continue 'dir; - } - - let file_name_lowercase: String = match entry_data.file_name().into_string() { - Ok(t) => t, - Err(_inspected) => { - warnings.push(flc!( - "core_file_not_utf8_name", - generate_translation_hashmap(vec![("name", entry_data.path().display().to_string())]) - )); - continue 'dir; - } - } - .to_lowercase(); - - if !self.allowed_extensions.matches_filename(&file_name_lowercase) { - continue 'dir; - } - - let current_file_name = current_folder.join(entry_data.file_name()); - if self.excluded_items.is_excluded(¤t_file_name) { - continue 'dir; - } - - let fe: FileEntry = FileEntry { - path: current_file_name.clone(), - size: metadata.len(), - modified_date: match metadata.modified() { - Ok(t) => match t.duration_since(UNIX_EPOCH) { - Ok(d) => d.as_secs(), - Err(_inspected) => { - warnings.push(flc!( - "core_file_modified_before_epoch", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string())]) - )); - 0 - } - }, - Err(e) => { - warnings.push(flc!( - "core_file_no_modification_date", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string()), ("reason", e.to_string())]) - )); - 0 - } - }, - }; - - fe_result.push((fe.size, fe)); + self.collect_file_entry(&atomic_counter, &metadata, entry_data, &mut fe_result, &mut warnings, current_folder); } } (dir_result, warnings, fe_result) @@ -327,12 +209,52 @@ impl BigFile { } } - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); - // Extract n biggest files to new TreeMap + self.extract_n_biggest_files(old_map); + Common::print_time(start_time, SystemTime::now(), "look_for_big_files"); + true + } + + pub fn collect_file_entry( + &self, + atomic_counter: &Arc, + metadata: &Metadata, + entry_data: &DirEntry, + fe_result: &mut Vec<(u64, FileEntry)>, + warnings: &mut Vec, + current_folder: &Path, + ) { + atomic_counter.fetch_add(1, Ordering::Relaxed); + + if metadata.len() == 0 { + return; + } + + let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else { + return; + }; + + if !self.allowed_extensions.matches_filename(&file_name_lowercase) { + return; + } + + let current_file_name = current_folder.join(entry_data.file_name()); + if self.excluded_items.is_excluded(¤t_file_name) { + return; + } + + let fe: FileEntry = FileEntry { + path: current_file_name.clone(), + size: metadata.len(), + modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false), + }; + + fe_result.push((fe.size, fe)); + } + + pub fn extract_n_biggest_files(&mut self, old_map: BTreeMap>) { let iter: Box>; if self.search_mode == SearchMode::SmallestFiles { iter = Box::new(old_map.into_iter()); @@ -360,9 +282,6 @@ impl BigFile { break; } } - - Common::print_time(start_time, SystemTime::now(), "look_for_big_files"); - true } pub fn set_number_of_files_to_check(&mut self, number_of_files_to_check: usize) { diff --git a/czkawka_core/src/broken_files.rs b/czkawka_core/src/broken_files.rs index 850093f..603c640 100644 --- a/czkawka_core/src/broken_files.rs +++ b/czkawka_core/src/broken_files.rs @@ -1,38 +1,32 @@ use std::collections::BTreeMap; -use std::fs::{File, Metadata}; +use std::fs::{DirEntry, File, Metadata}; use std::io::prelude::*; use std::io::{BufReader, BufWriter}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::sleep; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use std::{fs, mem, panic, thread}; +use std::time::SystemTime; +use std::{fs, mem, panic}; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; +use pdf::file::FileOptions; use pdf::object::ParseOptions; use pdf::PdfError; use pdf::PdfError::Try; use rayon::prelude::*; use serde::{Deserialize, Serialize}; -use crate::common::{create_crash_message, open_cache_folder, Common, LOOP_DURATION, PDF_FILES_EXTENSIONS}; +use crate::common::{ + check_folder_children, create_crash_message, open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common, PDF_FILES_EXTENSIONS, +}; use crate::common::{AUDIO_FILES_EXTENSIONS, IMAGE_RS_BROKEN_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS}; +use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; use crate::common_items::ExcludedItems; use crate::common_messages::Messages; use crate::common_traits::*; -use crate::flc; -use crate::localizer_core::generate_translation_hashmap; - -#[derive(Debug)] -pub struct ProgressData { - pub current_stage: u8, - pub max_stage: u8, - pub files_checked: usize, - pub files_to_check: usize, -} #[derive(Eq, PartialEq, Clone, Debug, Copy)] pub enum DeleteMethod { @@ -96,7 +90,8 @@ pub struct BrokenFiles { stopped_search: bool, checked_types: CheckedTypes, use_cache: bool, - delete_outdated_cache: bool, // TODO add this to GUI + // TODO add this to GUI + delete_outdated_cache: bool, save_also_as_json: bool, } @@ -121,7 +116,7 @@ impl BrokenFiles { } } - pub fn find_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.directories.optimize_directories(self.recursive_search, &mut self.text_messages); if !self.check_files(stop_receiver, progress_sender) { self.stopped_search = true; @@ -197,7 +192,7 @@ impl BrokenFiles { self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages); } - fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let start_time: SystemTime = SystemTime::now(); let mut folders_to_check: Vec = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector @@ -206,39 +201,13 @@ impl BrokenFiles { folders_to_check.push(id.clone()); } - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 0, - max_stage: 1, - files_checked: atomic_file_counter.load(Ordering::Relaxed), - files_to_check: 0, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 1, 0, CheckingMethod::None); while !folders_to_check.is_empty() { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); return false; } @@ -248,124 +217,31 @@ impl BrokenFiles { let mut dir_result = vec![]; let mut warnings = vec![]; let mut fe_result = vec![]; - // Read current dir children - let read_dir = match fs::read_dir(current_folder) { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_open_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - return (dir_result, warnings, fe_result); - } + + let Some(read_dir) = common_read_dir(current_folder, &mut warnings) else { + return (dir_result, warnings, fe_result); }; // Check every sub folder/file/link etc. - 'dir: for entry in read_dir { - let entry_data = match entry { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_entry_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } - }; - let metadata: Metadata = match entry_data.metadata() { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_metadata_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } + for entry in read_dir { + let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else { + continue; }; + if metadata.is_dir() { - if !self.recursive_search { - continue 'dir; - } - - let next_folder = current_folder.join(entry_data.file_name()); - if self.directories.is_excluded(&next_folder) { - continue 'dir; - } - - if self.excluded_items.is_excluded(&next_folder) { - continue 'dir; - } - - #[cfg(target_family = "unix")] - if self.directories.exclude_other_filesystems() { - match self.directories.is_on_other_filesystems(&next_folder) { - Ok(true) => continue 'dir, - Err(e) => warnings.push(e.to_string()), - _ => (), - } - } - - dir_result.push(next_folder); + check_folder_children( + &mut dir_result, + &mut warnings, + current_folder, + entry_data, + self.recursive_search, + &self.directories, + &self.excluded_items, + ); } else if metadata.is_file() { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - - let file_name_lowercase: String = match entry_data.file_name().into_string() { - Ok(t) => t, - Err(_inspected) => { - warnings.push(flc!( - "core_file_not_utf8_name", - generate_translation_hashmap(vec![("name", entry_data.path().display().to_string())]) - )); - continue 'dir; - } + if let Some(file_entry) = self.get_file_entry(&metadata, &atomic_counter, entry_data, &mut warnings, current_folder) { + fe_result.push((file_entry.path.to_string_lossy().to_string(), file_entry)); } - .to_lowercase(); - - if !self.allowed_extensions.matches_filename(&file_name_lowercase) { - continue 'dir; - } - - let type_of_file = check_extension_availability(&file_name_lowercase); - if type_of_file == TypeOfFile::Unknown { - continue 'dir; - } - - if !check_extension_allowed(&type_of_file, &self.checked_types) { - continue 'dir; - } - - let current_file_name = current_folder.join(entry_data.file_name()); - if self.excluded_items.is_excluded(¤t_file_name) { - continue 'dir; - } - - let fe: FileEntry = FileEntry { - path: current_file_name.clone(), - modified_date: match metadata.modified() { - Ok(t) => match t.duration_since(UNIX_EPOCH) { - Ok(d) => d.as_secs(), - Err(_inspected) => { - warnings.push(flc!( - "core_file_modified_before_epoch", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string())]) - )); - 0 - } - }, - Err(e) => { - warnings.push(flc!( - "core_file_no_modification_date", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string()), ("reason", e.to_string())]) - )); - 0 - } - }, - size: metadata.len(), - type_of_file, - error_string: String::new(), - }; - - fe_result.push((current_file_name.to_string_lossy().to_string(), fe)); } } (dir_result, warnings, fe_result) @@ -385,14 +261,150 @@ impl BrokenFiles { } } - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); Common::print_time(start_time, SystemTime::now(), "check_files"); true } - fn look_for_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn get_file_entry( + &self, + metadata: &Metadata, + atomic_counter: &Arc, + entry_data: &DirEntry, + warnings: &mut Vec, + current_folder: &Path, + ) -> Option { + atomic_counter.fetch_add(1, Ordering::Relaxed); + + let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else { + return None; + }; + + if !self.allowed_extensions.matches_filename(&file_name_lowercase) { + return None; + } + + let type_of_file = check_extension_availability(&file_name_lowercase); + if type_of_file == TypeOfFile::Unknown { + return None; + } + + if !check_extension_allowed(&type_of_file, &self.checked_types) { + return None; + } + + let current_file_name = current_folder.join(entry_data.file_name()); + if self.excluded_items.is_excluded(¤t_file_name) { + return None; + } + + let fe: FileEntry = FileEntry { + path: current_file_name.clone(), + modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false), + size: metadata.len(), + type_of_file, + error_string: String::new(), + }; + Some(fe) + } + + fn check_broken_image(&self, mut file_entry: FileEntry) -> Option { + let mut file_entry_clone = file_entry.clone(); + + let result = panic::catch_unwind(|| { + if let Err(e) = image::open(&file_entry.path) { + let error_string = e.to_string(); + // This error is a problem with image library, remove check when https://github.com/image-rs/jpeg-decoder/issues/130 will be fixed + if error_string.contains("spectral selection is not allowed in non-progressive scan") { + return None; + } + file_entry.error_string = error_string; + } + Some(file_entry) + }); + + // If image crashed during opening, needs to be printed info about crashes thing + if let Ok(image_result) = result { + image_result + } else { + let message = create_crash_message("Image-rs", &file_entry_clone.path.to_string_lossy(), "https://github.com/Serial-ATA/lofty-rs"); + println!("{message}"); + file_entry_clone.error_string = message; + Some(file_entry_clone) + } + } + fn check_broken_zip(&self, mut file_entry: FileEntry) -> Option { + match File::open(&file_entry.path) { + Ok(file) => { + if let Err(e) = zip::ZipArchive::new(file) { + file_entry.error_string = e.to_string(); + } + Some(file_entry) + } + Err(_inspected) => None, + } + } + fn check_broken_audio(&self, mut file_entry: FileEntry) -> Option { + match File::open(&file_entry.path) { + Ok(file) => { + let mut file_entry_clone = file_entry.clone(); + + let result = panic::catch_unwind(|| { + if let Err(e) = audio_checker::parse_audio_file(file) { + file_entry.error_string = e.to_string(); + } + Some(file_entry) + }); + + if let Ok(audio_result) = result { + audio_result + } else { + let message = create_crash_message("Symphonia", &file_entry_clone.path.to_string_lossy(), "https://github.com/pdeljanov/Symphonia"); + println!("{message}"); + file_entry_clone.error_string = message; + Some(file_entry_clone) + } + } + Err(_inspected) => None, + } + } + fn check_broken_pdf(&self, mut file_entry: FileEntry) -> Option { + let parser_options = ParseOptions::tolerant(); // Only show as broken files with really big bugs + + let mut file_entry_clone = file_entry.clone(); + let result = panic::catch_unwind(|| { + if let Err(e) = FileOptions::cached().parse_options(parser_options).open(&file_entry.path) { + if let PdfError::Io { .. } = e { + return None; + } + + let mut error_string = e.to_string(); + // Workaround for strange error message https://github.com/qarmin/czkawka/issues/898 + if error_string.starts_with("Try at") { + if let Some(start_index) = error_string.find("/pdf-") { + error_string = format!("Decoding error in pdf-rs library - {}", &error_string[start_index..]); + } + } + + file_entry.error_string = error_string; + let error = unpack_pdf_error(e); + if let PdfError::InvalidPassword = error { + return None; + } + } + Some(file_entry) + }); + if let Ok(pdf_result) = result { + pdf_result + } else { + let message = create_crash_message("PDF-rs", &file_entry_clone.path.to_string_lossy(), "https://github.com/pdf-rs/pdf"); + println!("{message}"); + file_entry_clone.error_string = message; + Some(file_entry_clone) + } + } + + fn look_for_broken_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let system_time = SystemTime::now(); let loaded_hash_map; @@ -430,134 +442,31 @@ impl BrokenFiles { non_cached_files_to_check = files_to_check; } - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common( + progress_sender, + &progress_thread_run, + &atomic_counter, + 1, + 1, + non_cached_files_to_check.len(), + CheckingMethod::None, + ); - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - let files_to_check = non_cached_files_to_check.len(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 1, - max_stage: 1, - files_checked: atomic_file_counter.load(Ordering::Relaxed), - files_to_check, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END let mut vec_file_entry: Vec = non_cached_files_to_check .into_par_iter() - .map(|(_, mut file_entry)| { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); + .map(|(_, file_entry)| { + atomic_counter.fetch_add(1, Ordering::Relaxed); if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { return None; } match file_entry.type_of_file { - TypeOfFile::Image => { - let mut file_entry_clone = file_entry.clone(); - - let result = panic::catch_unwind(|| { - if let Err(e) = image::open(&file_entry.path) { - let error_string = e.to_string(); - // This error is a problem with image library, remove check when https://github.com/image-rs/jpeg-decoder/issues/130 will be fixed - if error_string.contains("spectral selection is not allowed in non-progressive scan") { - return Some(None); - } - file_entry.error_string = error_string; - } - Some(Some(file_entry)) - }); - - // If image crashed during opening, needs to be printed info about crashes thing - if let Ok(image_result) = result { - image_result - } else { - let message = create_crash_message("Image-rs", &file_entry_clone.path.to_string_lossy(), "https://github.com/Serial-ATA/lofty-rs"); - println!("{message}"); - file_entry_clone.error_string = message; - Some(Some(file_entry_clone)) - } - } - TypeOfFile::ArchiveZip => match File::open(&file_entry.path) { - Ok(file) => { - if let Err(e) = zip::ZipArchive::new(file) { - file_entry.error_string = e.to_string(); - } - Some(Some(file_entry)) - } - Err(_inspected) => Some(None), - }, - TypeOfFile::Audio => match File::open(&file_entry.path) { - Ok(file) => { - let mut file_entry_clone = file_entry.clone(); - - let result = panic::catch_unwind(|| { - if let Err(e) = audio_checker::parse_audio_file(file) { - file_entry.error_string = e.to_string(); - } - Some(Some(file_entry)) - }); - - if let Ok(audio_result) = result { - audio_result - } else { - let message = create_crash_message("Symphonia", &file_entry_clone.path.to_string_lossy(), "https://github.com/pdeljanov/Symphonia"); - println!("{message}"); - file_entry_clone.error_string = message; - Some(Some(file_entry_clone)) - } - } - Err(_inspected) => Some(None), - }, - - TypeOfFile::PDF => match fs::read(&file_entry.path) { - Ok(content) => { - let parser_options = ParseOptions::tolerant(); // Only show as broken files with really big bugs - - let mut file_entry_clone = file_entry.clone(); - let result = panic::catch_unwind(|| { - if let Err(e) = pdf::file::File::from_data_with_options(content, parser_options) { - let mut error_string = e.to_string(); - // Workaround for strange error message https://github.com/qarmin/czkawka/issues/898 - if error_string.starts_with("Try at") { - if let Some(start_index) = error_string.find("/pdf-") { - error_string = format!("Decoding error in pdf-rs library - {}", &error_string[start_index..]); - } - } - - file_entry.error_string = error_string; - let error = unpack_pdf_error(e); - if let PdfError::InvalidPassword = error { - return Some(None); - } - } - Some(Some(file_entry)) - }); - if let Ok(pdf_result) = result { - pdf_result - } else { - let message = create_crash_message("PDF-rs", &file_entry_clone.path.to_string_lossy(), "https://github.com/pdf-rs/pdf"); - println!("{message}"); - file_entry_clone.error_string = message; - Some(Some(file_entry_clone)) - } - } - Err(_inspected) => Some(None), - }, - + TypeOfFile::Image => Some(self.check_broken_image(file_entry)), + TypeOfFile::ArchiveZip => Some(self.check_broken_zip(file_entry)), + TypeOfFile::Audio => Some(self.check_broken_audio(file_entry)), + TypeOfFile::PDF => Some(self.check_broken_pdf(file_entry)), // This means that cache read invalid value because maybe cache comes from different czkawka version TypeOfFile::Unknown => Some(None), } @@ -567,9 +476,7 @@ impl BrokenFiles { .map(Option::unwrap) .collect::>(); - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); // Just connect loaded results with already calculated for (_name, file_entry) in records_already_cached { @@ -603,6 +510,7 @@ impl BrokenFiles { true } + /// Function to delete files, from filed Vector fn delete_files(&mut self) { let start_time: SystemTime = SystemTime::now(); diff --git a/czkawka_core/src/common.rs b/czkawka_core/src/common.rs index 753c332..97f8d33 100644 --- a/czkawka_core/src/common.rs +++ b/czkawka_core/src/common.rs @@ -1,20 +1,28 @@ use std::ffi::OsString; -use std::fs; -use std::fs::{File, OpenOptions}; +use std::fs::{DirEntry, File, OpenOptions}; use std::io::BufReader; use std::path::{Path, PathBuf}; -use std::time::SystemTime; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::thread::{sleep, JoinHandle}; +use std::time::{Duration, SystemTime}; +use std::{fs, thread}; #[cfg(feature = "heif")] use anyhow::Result; use directories_next::ProjectDirs; +use futures::channel::mpsc::UnboundedSender; use image::{DynamicImage, ImageBuffer, Rgb}; use imagepipe::{ImageSource, Pipeline}; -// #[cfg(feature = "heif")] -// use libheif_rs::LibHeif; #[cfg(feature = "heif")] use libheif_rs::{ColorSpace, HeifContext, RgbChroma}; +// #[cfg(feature = "heif")] +// use libheif_rs::LibHeif; +use crate::common_dir_traversal::{CheckingMethod, ProgressData}; +use crate::common_directory::Directories; +use crate::common_items::ExcludedItems; + static NUMBER_OF_THREADS: state::Storage = state::Storage::new(); pub fn get_number_of_threads() -> usize { @@ -25,13 +33,16 @@ pub fn get_number_of_threads() -> usize { num_cpus::get() } } + pub fn set_default_number_of_threads() { set_number_of_threads(num_cpus::get()); } + #[must_use] pub fn get_default_number_of_threads() -> usize { num_cpus::get() } + pub fn set_number_of_threads(thread_number: usize) { NUMBER_OF_THREADS.set(thread_number); @@ -175,8 +186,8 @@ pub fn get_dynamic_image_from_raw_image(path: impl AsRef + std::fmt::Debug } }; - let Some(image) = ImageBuffer::, Vec>::from_raw(image.width as u32, image.height as u32, image.data) else { - return None; + let Some(image) = ImageBuffer::, Vec>::from_raw(image.width as u32, image.height as u32, image.data) else { + return None; }; // println!("Properly hashed {:?}", path); @@ -240,6 +251,7 @@ impl Common { } /// Function to check if directory match expression + #[must_use] pub fn regex_check(expression: &str, directory: impl AsRef) -> bool { if expression == "*" { return true; @@ -292,6 +304,7 @@ impl Common { true } + #[must_use] pub fn normalize_windows_path(path_to_change: impl AsRef) -> PathBuf { let path = path_to_change.as_ref(); @@ -317,6 +330,78 @@ impl Common { } } +pub fn check_folder_children( + dir_result: &mut Vec, + warnings: &mut Vec, + current_folder: &Path, + entry_data: &DirEntry, + recursive_search: bool, + directories: &Directories, + excluded_items: &ExcludedItems, +) { + if !recursive_search { + return; + } + + let next_folder = current_folder.join(entry_data.file_name()); + if directories.is_excluded(&next_folder) { + return; + } + + if excluded_items.is_excluded(&next_folder) { + return; + } + + #[cfg(target_family = "unix")] + if directories.exclude_other_filesystems() { + match directories.is_on_other_filesystems(&next_folder) { + Ok(true) => return, + Err(e) => warnings.push(e), + _ => (), + } + } + + dir_result.push(next_folder); +} + +pub fn prepare_thread_handler_common( + progress_sender: Option<&UnboundedSender>, + progress_thread_run: &Arc, + atomic_counter: &Arc, + current_stage: u8, + max_stage: u8, + max_value: usize, + checking_method: CheckingMethod, +) -> JoinHandle<()> { + if let Some(progress_sender) = progress_sender { + let progress_send = progress_sender.clone(); + let progress_thread_run = progress_thread_run.clone(); + let atomic_counter = atomic_counter.clone(); + thread::spawn(move || loop { + progress_send + .unbounded_send(ProgressData { + checking_method, + current_stage, + max_stage, + entries_checked: atomic_counter.load(Ordering::Relaxed), + entries_to_check: max_value, + }) + .unwrap(); + if !progress_thread_run.load(Ordering::Relaxed) { + break; + } + sleep(Duration::from_millis(LOOP_DURATION as u64)); + }) + } else { + thread::spawn(|| {}) + } +} + +pub fn send_info_and_wait_for_ending_all_threads(progress_thread_run: &Arc, progress_thread_handle: JoinHandle<()>) { + progress_thread_run.store(false, Ordering::Relaxed); + progress_thread_handle.join().unwrap(); +} + #[cfg(test)] mod test { use std::path::PathBuf; diff --git a/czkawka_core/src/common_dir_traversal.rs b/czkawka_core/src/common_dir_traversal.rs index bf18029..c318595 100644 --- a/czkawka_core/src/common_dir_traversal.rs +++ b/czkawka_core/src/common_dir_traversal.rs @@ -1,16 +1,16 @@ use std::collections::BTreeMap; -use std::fs::Metadata; +use std::fs; +use std::fs::{DirEntry, Metadata, ReadDir}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::sleep; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use std::{fs, thread}; +use std::time::{SystemTime, UNIX_EPOCH}; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use rayon::prelude::*; -use crate::common::LOOP_DURATION; +use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; use crate::common_items::ExcludedItems; @@ -100,7 +100,7 @@ pub struct DirTraversalBuilder<'a, 'b, F> { group_by: Option, root_dirs: Vec, stop_receiver: Option<&'a Receiver<()>>, - progress_sender: Option<&'b futures::channel::mpsc::UnboundedSender>, + progress_sender: Option<&'b UnboundedSender>, minimal_file_size: Option, maximal_file_size: Option, checking_method: CheckingMethod, @@ -116,7 +116,7 @@ pub struct DirTraversal<'a, 'b, F> { group_by: F, root_dirs: Vec, stop_receiver: Option<&'a Receiver<()>>, - progress_sender: Option<&'b futures::channel::mpsc::UnboundedSender>, + progress_sender: Option<&'b UnboundedSender>, recursive_search: bool, directories: Directories, excluded_items: ExcludedItems, @@ -169,7 +169,7 @@ impl<'a, 'b, F> DirTraversalBuilder<'a, 'b, F> { } #[must_use] - pub fn progress_sender(mut self, progress_sender: Option<&'b futures::channel::mpsc::UnboundedSender>) -> Self { + pub fn progress_sender(mut self, progress_sender: Option<&'b UnboundedSender>) -> Self { self.progress_sender = progress_sender; self } @@ -333,37 +333,9 @@ where // Add root folders for finding folders_to_check.extend(self.root_dirs); - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - - let atomic_entry_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = self.progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_entry_counter = atomic_entry_counter.clone(); - let checking_method = self.checking_method; - let max_stage = self.max_stage; - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - checking_method, - current_stage: 0, - max_stage, - entries_checked: atomic_entry_counter.load(Ordering::Relaxed), - entries_to_check: 0, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - - //// PROGRESS THREAD END + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common(self.progress_sender, &progress_thread_run, &atomic_counter, 0, self.max_stage, 0, self.checking_method); let DirTraversal { collect, @@ -379,9 +351,7 @@ where while !folders_to_check.is_empty() { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); return DirTraversalResult::Stopped; } @@ -393,174 +363,49 @@ where let mut fe_result = vec![]; let mut set_as_not_empty_folder_list = vec![]; let mut folder_entries_list = vec![]; - // Read current dir children - let read_dir = match fs::read_dir(current_folder) { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_open_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - return (dir_result, warnings, fe_result, set_as_not_empty_folder_list, folder_entries_list); - } + + let Some(read_dir) = common_read_dir(current_folder, &mut warnings) else { + return (dir_result, warnings, fe_result, set_as_not_empty_folder_list, folder_entries_list); }; // Check every sub folder/file/link etc. 'dir: for entry in read_dir { - let entry_data = match entry { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_entry_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } - }; - let metadata: Metadata = match entry_data.metadata() { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_metadata_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } + let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else { + continue; }; + match (entry_type(&metadata), collect) { (EntryType::Dir, Collect::Files | Collect::InvalidSymlinks) => { - if !recursive_search { - continue 'dir; - } - - let next_folder = current_folder.join(entry_data.file_name()); - if directories.is_excluded(&next_folder) { - continue 'dir; - } - - if excluded_items.is_excluded(&next_folder) { - continue 'dir; - } - - #[cfg(target_family = "unix")] - if directories.exclude_other_filesystems() { - match directories.is_on_other_filesystems(&next_folder) { - Ok(true) => continue 'dir, - Err(e) => warnings.push(e.to_string()), - _ => (), - } - } - - dir_result.push(next_folder); + process_dir_in_file_symlink_mode(recursive_search, current_folder, entry_data, &directories, &mut dir_result, &mut warnings, &excluded_items); } (EntryType::Dir, Collect::EmptyFolders) => { - atomic_entry_counter.fetch_add(1, Ordering::Relaxed); - let next_folder = current_folder.join(entry_data.file_name()); - if excluded_items.is_excluded(&next_folder) || directories.is_excluded(&next_folder) { - set_as_not_empty_folder_list.push(current_folder.clone()); - continue 'dir; - } - - #[cfg(target_family = "unix")] - if directories.exclude_other_filesystems() { - match directories.is_on_other_filesystems(&next_folder) { - Ok(true) => continue 'dir, - Err(e) => warnings.push(e.to_string()), - _ => (), - } - } - - dir_result.push(next_folder.clone()); - folder_entries_list.push(( - next_folder.clone(), - FolderEntry { - parent_path: Some(current_folder.clone()), - is_empty: FolderEmptiness::Maybe, - modified_date: match metadata.modified() { - Ok(t) => match t.duration_since(UNIX_EPOCH) { - Ok(d) => d.as_secs(), - Err(_inspected) => { - warnings.push(flc!( - "core_folder_modified_before_epoch", - generate_translation_hashmap(vec![("name", current_folder.display().to_string())]) - )); - 0 - } - }, - Err(e) => { - warnings.push(flc!( - "core_folder_no_modification_date", - generate_translation_hashmap(vec![("name", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - 0 - } - }, - }, - )); + atomic_counter.fetch_add(1, Ordering::Relaxed); + process_dir_in_dir_mode( + &metadata, + current_folder, + entry_data, + &directories, + &mut dir_result, + &mut warnings, + &excluded_items, + &mut set_as_not_empty_folder_list, + &mut folder_entries_list, + ); } (EntryType::File, Collect::Files) => { - atomic_entry_counter.fetch_add(1, Ordering::Relaxed); - - let file_name_lowercase: String = match entry_data.file_name().into_string() { - Ok(t) => t, - Err(_inspected) => { - warnings.push(flc!( - "core_file_not_utf8_name", - generate_translation_hashmap(vec![("name", entry_data.path().display().to_string())]) - )); - continue 'dir; - } - } - .to_lowercase(); - - if !allowed_extensions.matches_filename(&file_name_lowercase) { - continue 'dir; - } - - if (minimal_file_size..=maximal_file_size).contains(&metadata.len()) { - let current_file_name = current_folder.join(entry_data.file_name()); - if excluded_items.is_excluded(¤t_file_name) { - continue 'dir; - } - - #[cfg(target_family = "unix")] - if directories.exclude_other_filesystems() { - match directories.is_on_other_filesystems(¤t_file_name) { - Ok(true) => continue 'dir, - Err(e) => warnings.push(e.to_string()), - _ => (), - } - } - - // Creating new file entry - let fe: FileEntry = FileEntry { - path: current_file_name.clone(), - size: metadata.len(), - modified_date: match metadata.modified() { - Ok(t) => match t.duration_since(UNIX_EPOCH) { - Ok(d) => d.as_secs(), - Err(_inspected) => { - warnings.push(flc!( - "core_file_modified_before_epoch", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string())]) - )); - 0 - } - }, - Err(e) => { - warnings.push(flc!( - "core_file_no_modification_date", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string()), ("reason", e.to_string())]) - )); - 0 - } - }, - hash: String::new(), - symlink_info: None, - }; - - fe_result.push(fe); - } + atomic_counter.fetch_add(1, Ordering::Relaxed); + process_file_in_file_mode( + &metadata, + entry_data, + &mut warnings, + &mut fe_result, + &allowed_extensions, + current_folder, + &directories, + &excluded_items, + minimal_file_size, + maximal_file_size, + ); } (EntryType::File | EntryType::Symlink, Collect::EmptyFolders) => { #[cfg(target_family = "unix")] @@ -575,105 +420,20 @@ where set_as_not_empty_folder_list.push(current_folder.clone()); } (EntryType::File, Collect::InvalidSymlinks) => { - atomic_entry_counter.fetch_add(1, Ordering::Relaxed); + atomic_counter.fetch_add(1, Ordering::Relaxed); } (EntryType::Symlink, Collect::InvalidSymlinks) => { - atomic_entry_counter.fetch_add(1, Ordering::Relaxed); - - let file_name_lowercase: String = match entry_data.file_name().into_string() { - Ok(t) => t, - Err(_inspected) => { - warnings.push(flc!( - "core_file_not_utf8_name", - generate_translation_hashmap(vec![("name", entry_data.path().display().to_string())]) - )); - continue 'dir; - } - } - .to_lowercase(); - - if !allowed_extensions.matches_filename(&file_name_lowercase) { - continue 'dir; - } - - let current_file_name = current_folder.join(entry_data.file_name()); - if excluded_items.is_excluded(¤t_file_name) { - continue 'dir; - } - - #[cfg(target_family = "unix")] - if directories.exclude_other_filesystems() { - match directories.is_on_other_filesystems(current_folder) { - Ok(true) => continue 'dir, - Err(e) => warnings.push(e.to_string()), - _ => (), - } - } - - let mut destination_path = PathBuf::new(); - let type_of_error; - - match current_file_name.read_link() { - Ok(t) => { - destination_path.push(t); - let mut number_of_loop = 0; - let mut current_path = current_file_name.clone(); - loop { - if number_of_loop == 0 && !current_path.exists() { - type_of_error = ErrorType::NonExistentFile; - break; - } - if number_of_loop == MAX_NUMBER_OF_SYMLINK_JUMPS { - type_of_error = ErrorType::InfiniteRecursion; - break; - } - - current_path = match current_path.read_link() { - Ok(t) => t, - Err(_inspected) => { - // Looks that some next symlinks are broken, but we do nothing with it - TODO why they are broken - continue 'dir; - } - }; - - number_of_loop += 1; - } - } - Err(_inspected) => { - // Failed to load info about it - type_of_error = ErrorType::NonExistentFile; - } - } - - // Creating new file entry - let fe: FileEntry = FileEntry { - path: current_file_name.clone(), - modified_date: match metadata.modified() { - Ok(t) => match t.duration_since(UNIX_EPOCH) { - Ok(d) => d.as_secs(), - Err(_inspected) => { - warnings.push(flc!( - "core_file_modified_before_epoch", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string())]) - )); - 0 - } - }, - Err(e) => { - warnings.push(flc!( - "core_file_no_modification_date", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string()), ("reason", e.to_string())]) - )); - 0 - } - }, - size: 0, - hash: String::new(), - symlink_info: Some(SymlinkInfo { destination_path, type_of_error }), - }; - - // Adding files to Vector - fe_result.push(fe); + atomic_counter.fetch_add(1, Ordering::Relaxed); + process_symlink_in_symlink_mode( + &metadata, + entry_data, + &mut warnings, + &mut fe_result, + &allowed_extensions, + current_folder, + &directories, + &excluded_items, + ); } (EntryType::Symlink, Collect::Files) | (EntryType::Other, _) => { // nothing to do @@ -704,9 +464,7 @@ where } } - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); match collect { Collect::Files | Collect::InvalidSymlinks => DirTraversalResult::SuccessFiles { @@ -723,10 +481,287 @@ where } } +fn process_file_in_file_mode( + metadata: &Metadata, + entry_data: &DirEntry, + warnings: &mut Vec, + fe_result: &mut Vec, + allowed_extensions: &Extensions, + current_folder: &Path, + directories: &Directories, + excluded_items: &ExcludedItems, + minimal_file_size: u64, + maximal_file_size: u64, +) { + let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else { + return; + }; + + if !allowed_extensions.matches_filename(&file_name_lowercase) { + return; + } + + if (minimal_file_size..=maximal_file_size).contains(&metadata.len()) { + let current_file_name = current_folder.join(entry_data.file_name()); + if excluded_items.is_excluded(¤t_file_name) { + return; + } + + #[cfg(target_family = "unix")] + if directories.exclude_other_filesystems() { + match directories.is_on_other_filesystems(¤t_file_name) { + Ok(true) => return, + Err(e) => warnings.push(e), + _ => (), + } + } + + // Creating new file entry + let fe: FileEntry = FileEntry { + path: current_file_name.clone(), + size: metadata.len(), + modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false), + hash: String::new(), + symlink_info: None, + }; + + fe_result.push(fe); + } +} + +fn process_dir_in_dir_mode( + metadata: &Metadata, + current_folder: &Path, + entry_data: &DirEntry, + directories: &Directories, + dir_result: &mut Vec, + warnings: &mut Vec, + excluded_items: &ExcludedItems, + set_as_not_empty_folder_list: &mut Vec, + folder_entries_list: &mut Vec<(PathBuf, FolderEntry)>, +) { + let next_folder = current_folder.join(entry_data.file_name()); + if excluded_items.is_excluded(&next_folder) || directories.is_excluded(&next_folder) { + set_as_not_empty_folder_list.push(current_folder.to_path_buf()); + return; + } + + #[cfg(target_family = "unix")] + if directories.exclude_other_filesystems() { + match directories.is_on_other_filesystems(&next_folder) { + Ok(true) => return, + Err(e) => warnings.push(e), + _ => (), + } + } + + dir_result.push(next_folder.clone()); + folder_entries_list.push(( + next_folder, + FolderEntry { + parent_path: Some(current_folder.to_path_buf()), + is_empty: FolderEmptiness::Maybe, + modified_date: get_modified_time(metadata, warnings, current_folder, true), + }, + )); +} + +fn process_dir_in_file_symlink_mode( + recursive_search: bool, + current_folder: &Path, + entry_data: &DirEntry, + directories: &Directories, + dir_result: &mut Vec, + warnings: &mut Vec, + excluded_items: &ExcludedItems, +) { + if !recursive_search { + return; + } + + let next_folder = current_folder.join(entry_data.file_name()); + if directories.is_excluded(&next_folder) { + return; + } + + if excluded_items.is_excluded(&next_folder) { + return; + } + + #[cfg(target_family = "unix")] + if directories.exclude_other_filesystems() { + match directories.is_on_other_filesystems(&next_folder) { + Ok(true) => return, + Err(e) => warnings.push(e), + _ => (), + } + } + + dir_result.push(next_folder); +} + +fn process_symlink_in_symlink_mode( + metadata: &Metadata, + entry_data: &DirEntry, + warnings: &mut Vec, + fe_result: &mut Vec, + allowed_extensions: &Extensions, + current_folder: &Path, + directories: &Directories, + excluded_items: &ExcludedItems, +) { + let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else { + return; + }; + + if !allowed_extensions.matches_filename(&file_name_lowercase) { + return; + } + + let current_file_name = current_folder.join(entry_data.file_name()); + if excluded_items.is_excluded(¤t_file_name) { + return; + } + + #[cfg(target_family = "unix")] + if directories.exclude_other_filesystems() { + match directories.is_on_other_filesystems(current_folder) { + Ok(true) => return, + Err(e) => warnings.push(e), + _ => (), + } + } + + let mut destination_path = PathBuf::new(); + let type_of_error; + + match current_file_name.read_link() { + Ok(t) => { + destination_path.push(t); + let mut number_of_loop = 0; + let mut current_path = current_file_name.clone(); + loop { + if number_of_loop == 0 && !current_path.exists() { + type_of_error = ErrorType::NonExistentFile; + break; + } + if number_of_loop == MAX_NUMBER_OF_SYMLINK_JUMPS { + type_of_error = ErrorType::InfiniteRecursion; + break; + } + + current_path = match current_path.read_link() { + Ok(t) => t, + Err(_inspected) => { + // Looks that some next symlinks are broken, but we do nothing with it - TODO why they are broken + return; + } + }; + + number_of_loop += 1; + } + } + Err(_inspected) => { + // Failed to load info about it + type_of_error = ErrorType::NonExistentFile; + } + } + + // Creating new file entry + let fe: FileEntry = FileEntry { + path: current_file_name.clone(), + modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false), + size: 0, + hash: String::new(), + symlink_info: Some(SymlinkInfo { destination_path, type_of_error }), + }; + + // Adding files to Vector + fe_result.push(fe); +} + +pub fn common_read_dir(current_folder: &Path, warnings: &mut Vec) -> Option { + match fs::read_dir(current_folder) { + Ok(t) => Some(t), + Err(e) => { + warnings.push(flc!( + "core_cannot_open_dir", + generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) + )); + None + } + } +} + +pub fn common_get_entry_data_metadata<'a>(entry: &'a Result, warnings: &mut Vec, current_folder: &Path) -> Option<(&'a DirEntry, Metadata)> { + let entry_data = match entry { + Ok(t) => t, + Err(e) => { + warnings.push(flc!( + "core_cannot_read_entry_dir", + generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) + )); + return None; + } + }; + let metadata: Metadata = match entry_data.metadata() { + Ok(t) => t, + Err(e) => { + warnings.push(flc!( + "core_cannot_read_metadata_dir", + generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) + )); + return None; + } + }; + Some((entry_data, metadata)) +} + +pub fn get_modified_time(metadata: &Metadata, warnings: &mut Vec, current_file_name: &Path, is_folder: bool) -> u64 { + match metadata.modified() { + Ok(t) => match t.duration_since(UNIX_EPOCH) { + Ok(d) => d.as_secs(), + Err(_inspected) => { + let translation_hashmap = generate_translation_hashmap(vec![("name", current_file_name.display().to_string())]); + if is_folder { + warnings.push(flc!("core_folder_modified_before_epoch", translation_hashmap)); + } else { + warnings.push(flc!("core_file_modified_before_epoch", translation_hashmap)); + } + 0 + } + }, + Err(e) => { + let translation_hashmap = generate_translation_hashmap(vec![("name", current_file_name.display().to_string()), ("reason", e.to_string())]); + if is_folder { + warnings.push(flc!("core_folder_no_modification_date", translation_hashmap)); + } else { + warnings.push(flc!("core_file_no_modification_date", translation_hashmap)); + } + 0 + } + } +} + +pub fn get_lowercase_name(entry_data: &DirEntry, warnings: &mut Vec) -> Option { + let name = match entry_data.file_name().into_string() { + Ok(t) => t, + Err(_inspected) => { + warnings.push(flc!( + "core_file_not_utf8_name", + generate_translation_hashmap(vec![("name", entry_data.path().display().to_string())]) + )); + return None; + } + } + .to_lowercase(); + Some(name) +} + fn set_as_not_empty_folder(folder_entries: &mut BTreeMap, current_folder: &Path) { - // Not folder so it may be a file or symbolic link so it isn't empty - folder_entries.get_mut(current_folder).unwrap().is_empty = FolderEmptiness::No; let mut d = folder_entries.get_mut(current_folder).unwrap(); + // Not folder so it may be a file or symbolic link so it isn't empty + d.is_empty = FolderEmptiness::No; // Loop to recursively set as non empty this and all his parent folders loop { d.is_empty = FolderEmptiness::No; diff --git a/czkawka_core/src/duplicate.rs b/czkawka_core/src/duplicate.rs index 77edae1..33790f1 100644 --- a/czkawka_core/src/duplicate.rs +++ b/czkawka_core/src/duplicate.rs @@ -11,16 +11,17 @@ use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::{sleep, JoinHandle}; -use std::time::{Duration, SystemTime}; -use std::{fs, mem, thread}; +use std::time::SystemTime; +use std::{fs, mem}; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use humansize::format_size; use humansize::BINARY; use rayon::prelude::*; +use xxhash_rust::xxh3::Xxh3; -use crate::common::{open_cache_folder, Common, LOOP_DURATION}; +use crate::common::{open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; @@ -44,7 +45,7 @@ impl HashType { match self { HashType::Blake3 => Box::new(blake3::Hasher::new()), HashType::Crc32 => Box::new(crc32fast::Hasher::new()), - HashType::Xxh3 => Box::new(xxhash_rust::xxh3::Xxh3::new()), + HashType::Xxh3 => Box::new(Xxh3::new()), } } } @@ -148,7 +149,7 @@ impl DuplicateFinder { } } - pub fn find_duplicates(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_duplicates(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.directories.optimize_directories(self.recursive_search, &mut self.text_messages); self.use_reference_folders = !self.directories.reference_directories.is_empty(); @@ -341,7 +342,7 @@ impl DuplicateFinder { &self.files_with_identical_size_names_referenced } - fn check_files_name(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files_name(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let group_by_func = if self.case_sensitive_name_comparison { |fe: &FileEntry| fe.path.file_name().unwrap().to_string_lossy().to_string() } else { @@ -436,7 +437,7 @@ impl DuplicateFinder { } } - fn check_files_size_name(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files_size_name(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let group_by_func = if self.case_sensitive_name_comparison { |fe: &FileEntry| (fe.size, fe.path.file_name().unwrap().to_string_lossy().to_string()) } else { @@ -536,7 +537,7 @@ impl DuplicateFinder { /// Read file length and puts it to different boxes(each for different lengths) /// If in box is only 1 result, then it is removed - fn check_files_size(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files_size(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let max_stage = match self.check_method { CheckingMethod::Size => 0, CheckingMethod::Hash => 2, @@ -644,70 +645,15 @@ impl DuplicateFinder { } } - // TODO Generalize this if possible with different tools - fn prepare_hash_thread_handler( - &self, - progress_sender: Option<&futures::channel::mpsc::UnboundedSender>, - progress_thread_run: Arc, - atomic_counter: Arc, - current_stage: u8, - max_stage: u8, - max_value: usize, - ) -> JoinHandle<()> { - if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run; - let atomic_counter = atomic_counter; - let checking_method = self.check_method; - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - checking_method, - current_stage, - max_stage, - entries_checked: atomic_counter.load(Ordering::Relaxed), - entries_to_check: max_value, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - } - } - - fn prehashing( - &mut self, - stop_receiver: Option<&Receiver<()>>, - progress_sender: Option<&futures::channel::mpsc::UnboundedSender>, - pre_checked_map: &mut BTreeMap>, - ) -> Option<()> { - let start_time: SystemTime = SystemTime::now(); - let check_type = self.hash_type; - let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread - - let progress_thread_run = Arc::new(AtomicBool::new(true)); - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - let progress_thread_handle = self.prepare_hash_thread_handler( - progress_sender, - progress_thread_run.clone(), - atomic_file_counter.clone(), - 1, - 2, - self.files_with_identical_size.values().map(Vec::len).sum(), - ); - - let loaded_hash_map; - let mut records_already_cached: BTreeMap> = Default::default(); - let mut non_cached_files_to_check: BTreeMap> = Default::default(); - + fn prehash_load_cache_at_start(&mut self) -> (BTreeMap>, BTreeMap>, BTreeMap>) { // Cache algorithm // - Load data from cache // - Convert from BT> to BT // - Save to proper values + let loaded_hash_map; + let mut records_already_cached: BTreeMap> = Default::default(); + let mut non_cached_files_to_check: BTreeMap> = Default::default(); + if self.use_prehash_cache { loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache, &self.hash_type, true) { Some(t) => t, @@ -741,6 +687,59 @@ impl DuplicateFinder { loaded_hash_map = Default::default(); mem::swap(&mut self.files_with_identical_size, &mut non_cached_files_to_check); } + (loaded_hash_map, records_already_cached, non_cached_files_to_check) + } + + fn prehash_save_cache_at_exit(&mut self, loaded_hash_map: BTreeMap>, pre_hash_results: &Vec<(u64, BTreeMap>, Vec)>) { + if self.use_prehash_cache { + // All results = records already cached + computed results + let mut save_cache_to_hashmap: BTreeMap = Default::default(); + + for (size, vec_file_entry) in loaded_hash_map { + if size >= self.minimal_prehash_cache_file_size { + for file_entry in vec_file_entry { + save_cache_to_hashmap.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone()); + } + } + } + + for (size, hash_map, _errors) in pre_hash_results { + if *size >= self.minimal_prehash_cache_file_size { + for vec_file_entry in hash_map.values() { + for file_entry in vec_file_entry { + save_cache_to_hashmap.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone()); + } + } + } + } + + save_hashes_to_file(&save_cache_to_hashmap, &mut self.text_messages, &self.hash_type, true, self.minimal_prehash_cache_file_size); + } + } + + fn prehashing( + &mut self, + stop_receiver: Option<&Receiver<()>>, + progress_sender: Option<&UnboundedSender>, + pre_checked_map: &mut BTreeMap>, + ) -> Option<()> { + let start_time: SystemTime = SystemTime::now(); + let check_type = self.hash_type; + let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread + + let progress_thread_run = Arc::new(AtomicBool::new(true)); + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common( + progress_sender, + &progress_thread_run, + &atomic_counter, + 1, + 2, + self.files_with_identical_size.values().map(Vec::len).sum(), + self.check_method, + ); + + let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.prehash_load_cache_at_start(); #[allow(clippy::type_complexity)] let pre_hash_results: Vec<(u64, BTreeMap>, Vec)> = non_cached_files_to_check @@ -750,7 +749,7 @@ impl DuplicateFinder { let mut errors: Vec = Vec::new(); let mut buffer = [0u8; 1024 * 2]; - atomic_file_counter.fetch_add(vec_file_entry.len(), Ordering::Relaxed); + atomic_counter.fetch_add(vec_file_entry.len(), Ordering::Relaxed); for file_entry in vec_file_entry { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { check_was_stopped.store(true, Ordering::Relaxed); @@ -768,9 +767,7 @@ impl DuplicateFinder { .while_some() .collect(); - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); // Check if user aborted search(only from GUI) if check_was_stopped.load(Ordering::Relaxed) { @@ -792,114 +789,138 @@ impl DuplicateFinder { } } - if self.use_prehash_cache { - // All results = records already cached + computed results - let mut save_cache_to_hashmap: BTreeMap = Default::default(); - - for (size, vec_file_entry) in loaded_hash_map { - if size >= self.minimal_prehash_cache_file_size { - for file_entry in vec_file_entry { - save_cache_to_hashmap.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone()); - } - } - } - - for (size, hash_map, _errors) in &pre_hash_results { - if *size >= self.minimal_prehash_cache_file_size { - for vec_file_entry in hash_map.values() { - for file_entry in vec_file_entry { - save_cache_to_hashmap.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone()); - } - } - } - } - - save_hashes_to_file(&save_cache_to_hashmap, &mut self.text_messages, &self.hash_type, true, self.minimal_prehash_cache_file_size); - } + self.prehash_save_cache_at_exit(loaded_hash_map, &pre_hash_results); Common::print_time(start_time, SystemTime::now(), "check_files_hash - prehash"); Some(()) } + fn full_hashing_load_cache_at_start( + &mut self, + mut pre_checked_map: BTreeMap>, + ) -> (BTreeMap>, BTreeMap>, BTreeMap>) { + let loaded_hash_map; + let mut records_already_cached: BTreeMap> = Default::default(); + let mut non_cached_files_to_check: BTreeMap> = Default::default(); + + if self.use_cache { + loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache, &self.hash_type, false) { + Some(t) => t, + None => Default::default(), + }; + + for (size, vec_file_entry) in pre_checked_map { + #[allow(clippy::collapsible_if)] + if !loaded_hash_map.contains_key(&size) { + // If loaded data doesn't contains current info + non_cached_files_to_check.insert(size, vec_file_entry); + } else { + let loaded_vec_file_entry = loaded_hash_map.get(&size).unwrap(); + + for file_entry in vec_file_entry { + let mut found: bool = false; + for loaded_file_entry in loaded_vec_file_entry { + if file_entry.path == loaded_file_entry.path && file_entry.modified_date == loaded_file_entry.modified_date { + records_already_cached.entry(file_entry.size).or_insert_with(Vec::new).push(loaded_file_entry.clone()); + found = true; + break; + } + } + + if !found { + non_cached_files_to_check.entry(file_entry.size).or_insert_with(Vec::new).push(file_entry); + } + } + } + } + } else { + loaded_hash_map = Default::default(); + mem::swap(&mut pre_checked_map, &mut non_cached_files_to_check); + } + (loaded_hash_map, records_already_cached, non_cached_files_to_check) + } + + fn full_hashing_save_cache_at_exit( + &mut self, + records_already_cached: BTreeMap>, + full_hash_results: &mut Vec<(u64, BTreeMap>, Vec)>, + loaded_hash_map: BTreeMap>, + ) { + if !self.use_cache { + return; + } + 'main: for (size, vec_file_entry) in records_already_cached { + // Check if size already exists, if exists we must to change it outside because cannot have mut and non mut reference to full_hash_results + for (full_size, full_hashmap, _errors) in &mut (*full_hash_results) { + if size == *full_size { + for file_entry in vec_file_entry { + full_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry); + } + continue 'main; + } + } + // Size doesn't exists add results to files + let mut temp_hashmap: BTreeMap> = Default::default(); + for file_entry in vec_file_entry { + temp_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry); + } + full_hash_results.push((size, temp_hashmap, Vec::new())); + } + + // Must save all results to file, old loaded from file with all currently counted results + let mut all_results: BTreeMap = Default::default(); + for (_size, vec_file_entry) in loaded_hash_map { + for file_entry in vec_file_entry { + all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry); + } + } + for (_size, hashmap, _errors) in full_hash_results { + for vec_file_entry in hashmap.values() { + for file_entry in vec_file_entry { + all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone()); + } + } + } + save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type, false, self.minimal_cache_file_size); + } + fn full_hashing( &mut self, stop_receiver: Option<&Receiver<()>>, - progress_sender: Option<&futures::channel::mpsc::UnboundedSender>, - mut pre_checked_map: BTreeMap>, + progress_sender: Option<&UnboundedSender>, + pre_checked_map: BTreeMap>, ) -> Option<()> { let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread let check_type = self.hash_type; let start_time: SystemTime = SystemTime::now(); - //// PROGRESS THREAD START - let progress_thread_run = Arc::new(AtomicBool::new(true)); - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - let progress_thread_handle = self.prepare_hash_thread_handler( + let progress_thread_run = Arc::new(AtomicBool::new(true)); + let atomic_counter = Arc::new(AtomicUsize::new(0)); + + let progress_thread_handle = prepare_thread_handler_common( progress_sender, - progress_thread_run.clone(), - atomic_file_counter.clone(), + &progress_thread_run, + &atomic_counter, 2, 2, pre_checked_map.values().map(Vec::len).sum(), + self.check_method, ); - //// PROGRESS THREAD END - ///////////////////////////////////////////////////////////////////////////// HASHING START { - #[allow(clippy::type_complexity)] - let mut full_hash_results: Vec<(u64, BTreeMap>, Vec)>; + let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.full_hashing_load_cache_at_start(pre_checked_map); - let loaded_hash_map; - - let mut records_already_cached: BTreeMap> = Default::default(); - let mut non_cached_files_to_check: BTreeMap> = Default::default(); - - if self.use_cache { - loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache, &self.hash_type, false) { - Some(t) => t, - None => Default::default(), - }; - - for (size, vec_file_entry) in pre_checked_map { - #[allow(clippy::collapsible_if)] - if !loaded_hash_map.contains_key(&size) { - // If loaded data doesn't contains current info - non_cached_files_to_check.insert(size, vec_file_entry); - } else { - let loaded_vec_file_entry = loaded_hash_map.get(&size).unwrap(); - - for file_entry in vec_file_entry { - let mut found: bool = false; - for loaded_file_entry in loaded_vec_file_entry { - if file_entry.path == loaded_file_entry.path && file_entry.modified_date == loaded_file_entry.modified_date { - records_already_cached.entry(file_entry.size).or_insert_with(Vec::new).push(loaded_file_entry.clone()); - found = true; - break; - } - } - - if !found { - non_cached_files_to_check.entry(file_entry.size).or_insert_with(Vec::new).push(file_entry); - } - } - } - } - } else { - loaded_hash_map = Default::default(); - mem::swap(&mut pre_checked_map, &mut non_cached_files_to_check); - } - - full_hash_results = non_cached_files_to_check + let mut full_hash_results: Vec<(u64, BTreeMap>, Vec)> = non_cached_files_to_check .into_par_iter() .map(|(size, vec_file_entry)| { let mut hashmap_with_hash: BTreeMap> = Default::default(); let mut errors: Vec = Vec::new(); let mut buffer = [0u8; 1024 * 16]; - atomic_file_counter.fetch_add(vec_file_entry.len(), Ordering::Relaxed); + atomic_counter.fetch_add(vec_file_entry.len(), Ordering::Relaxed); for mut file_entry in vec_file_entry { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { check_was_stopped.store(true, Ordering::Relaxed); @@ -919,45 +940,9 @@ impl DuplicateFinder { .while_some() .collect(); - if self.use_cache { - 'main: for (size, vec_file_entry) in records_already_cached { - // Check if size already exists, if exists we must to change it outside because cannot have mut and non mut reference to full_hash_results - for (full_size, full_hashmap, _errors) in &mut full_hash_results { - if size == *full_size { - for file_entry in vec_file_entry { - full_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry); - } - continue 'main; - } - } - // Size doesn't exists add results to files - let mut temp_hashmap: BTreeMap> = Default::default(); - for file_entry in vec_file_entry { - temp_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry); - } - full_hash_results.push((size, temp_hashmap, Vec::new())); - } + self.full_hashing_save_cache_at_exit(records_already_cached, &mut full_hash_results, loaded_hash_map); - // Must save all results to file, old loaded from file with all currently counted results - let mut all_results: BTreeMap = Default::default(); - for (_size, vec_file_entry) in loaded_hash_map { - for file_entry in vec_file_entry { - all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry); - } - } - for (_size, hashmap, _errors) in &full_hash_results { - for vec_file_entry in hashmap.values() { - for file_entry in vec_file_entry { - all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone()); - } - } - } - save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type, false, self.minimal_cache_file_size); - } - - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); // Break if stop was clicked after saving to cache if check_was_stopped.load(Ordering::Relaxed) { @@ -1035,7 +1020,7 @@ impl DuplicateFinder { } /// The slowest checking type, which must be applied after checking for size - fn check_files_hash(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files_hash(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { assert_eq!(self.check_method, CheckingMethod::Hash); let mut pre_checked_map: BTreeMap> = Default::default(); @@ -1509,7 +1494,7 @@ pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache open_cache_folder(&get_file_hash_name(type_of_hash, is_prehash), false, false, &mut text_messages.warnings) { // Unwrap could fail when failed to open cache file, but json would exists - let Some(file_handler) = file_handler else { return Default::default() }; + let Some(file_handler) = file_handler else { return Default::default(); }; let reader = BufReader::new(file_handler); let mut hashmap_loaded_entries: BTreeMap> = Default::default(); @@ -1634,7 +1619,7 @@ impl MyHasher for crc32fast::Hasher { } } -impl MyHasher for xxhash_rust::xxh3::Xxh3 { +impl MyHasher for Xxh3 { fn update(&mut self, bytes: &[u8]) { self.write(bytes); } diff --git a/czkawka_core/src/empty_files.rs b/czkawka_core/src/empty_files.rs index b3bcba2..f1f76c3 100644 --- a/czkawka_core/src/empty_files.rs +++ b/czkawka_core/src/empty_files.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use std::time::SystemTime; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use crate::common::Common; use crate::common_dir_traversal::{DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData}; @@ -64,7 +65,7 @@ impl EmptyFiles { } /// Finding empty files, save results to internal struct variables - pub fn find_empty_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_empty_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.directories.optimize_directories(self.recursive_search, &mut self.text_messages); if !self.check_files(stop_receiver, progress_sender) { self.stopped_search = true; @@ -125,7 +126,7 @@ impl EmptyFiles { } /// Check files for any with size == 0 - fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let result = DirTraversalBuilder::new() .root_dirs(self.directories.included_directories.clone()) .group_by(|_fe| ()) diff --git a/czkawka_core/src/empty_folder.rs b/czkawka_core/src/empty_folder.rs index 0ab2c38..f4129df 100644 --- a/czkawka_core/src/empty_folder.rs +++ b/czkawka_core/src/empty_folder.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use std::time::SystemTime; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use crate::common::Common; use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, FolderEmptiness, FolderEntry, ProgressData}; @@ -88,7 +89,7 @@ impl EmptyFolder { self.directories.set_excluded_directory(excluded_directory, &mut self.text_messages); } /// Public function used by CLI to search for empty folders - pub fn find_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.directories.optimize_directories(true, &mut self.text_messages); if !self.check_for_empty_folders(stop_receiver, progress_sender) { self.stopped_search = true; @@ -128,7 +129,7 @@ impl EmptyFolder { /// Function to check if folder are empty. /// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty - fn check_for_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_for_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let result = DirTraversalBuilder::new() .root_dirs(self.directories.included_directories.clone()) .group_by(|_fe| ()) diff --git a/czkawka_core/src/invalid_symlinks.rs b/czkawka_core/src/invalid_symlinks.rs index ba1a8c8..1bc064b 100644 --- a/czkawka_core/src/invalid_symlinks.rs +++ b/czkawka_core/src/invalid_symlinks.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use std::time::SystemTime; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use crate::common::Common; use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, ErrorType, FileEntry, ProgressData}; @@ -63,7 +64,7 @@ impl InvalidSymlinks { } } - pub fn find_invalid_links(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_invalid_links(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.directories.optimize_directories(self.recursive_search, &mut self.text_messages); if !self.check_files(stop_receiver, progress_sender) { self.stopped_search = true; @@ -124,7 +125,7 @@ impl InvalidSymlinks { } /// Check files for any with size == 0 - fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let result = DirTraversalBuilder::new() .root_dirs(self.directories.included_directories.clone()) .group_by(|_fe| ()) diff --git a/czkawka_core/src/lib.rs b/czkawka_core/src/lib.rs index e37b1f5..ddbb975 100644 --- a/czkawka_core/src/lib.rs +++ b/czkawka_core/src/lib.rs @@ -1,6 +1,7 @@ #![allow(clippy::collapsible_else_if)] #![allow(clippy::type_complexity)] #![allow(clippy::needless_late_init)] +#![allow(clippy::too_many_arguments)] #[macro_use] extern crate bitflags; diff --git a/czkawka_core/src/same_music.rs b/czkawka_core/src/same_music.rs index 83ccf5d..5af0fad 100644 --- a/czkawka_core/src/same_music.rs +++ b/czkawka_core/src/same_music.rs @@ -5,18 +5,18 @@ use std::io::{BufReader, BufWriter}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::sleep; -use std::time::{Duration, SystemTime}; -use std::{mem, panic, thread}; +use std::time::SystemTime; +use std::{mem, panic}; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use lofty::TaggedFileExt; use lofty::{read_from, AudioFile, ItemKey}; use rayon::prelude::*; use serde::{Deserialize, Serialize}; -use crate::common::{create_crash_message, AUDIO_FILES_EXTENSIONS}; -use crate::common::{open_cache_folder, Common, LOOP_DURATION}; +use crate::common::{create_crash_message, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS}; +use crate::common::{open_cache_folder, Common}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; @@ -30,6 +30,12 @@ pub enum DeleteMethod { Delete, } +#[derive(Eq, PartialEq, Clone, Debug, Copy)] +pub enum AudioCheckMethod { + Tags, + Content, +} + bitflags! { #[derive(PartialEq, Copy, Clone, Debug)] pub struct MusicSimilarity : u32 { @@ -112,6 +118,7 @@ pub struct SameMusic { delete_outdated_cache: bool, // TODO add this to GUI use_reference_folders: bool, save_also_as_json: bool, + check_type: AudioCheckMethod, } impl SameMusic { @@ -138,23 +145,31 @@ impl SameMusic { use_reference_folders: false, duplicated_music_entries_referenced: vec![], save_also_as_json: false, + check_type: AudioCheckMethod::Tags, } } - pub fn find_same_music(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_same_music(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.directories.optimize_directories(self.recursive_search, &mut self.text_messages); self.use_reference_folders = !self.directories.reference_directories.is_empty(); if !self.check_files(stop_receiver, progress_sender) { self.stopped_search = true; return; } - if !self.check_records_multithreaded(stop_receiver, progress_sender) { - self.stopped_search = true; - return; - } - if !self.check_for_duplicates(stop_receiver, progress_sender) { - self.stopped_search = true; - return; + match self.check_type { + AudioCheckMethod::Tags => { + if !self.read_tags(stop_receiver, progress_sender) { + self.stopped_search = true; + return; + } + if !self.check_for_duplicate_tags(stop_receiver, progress_sender) { + self.stopped_search = true; + return; + } + } + AudioCheckMethod::Content => { + unimplemented!(); + } } self.delete_files(); self.debug_print(); @@ -262,7 +277,7 @@ impl SameMusic { self.use_reference_folders } - fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { if !self.allowed_extensions.using_custom_extensions() { self.allowed_extensions.extend_allowed_extensions(AUDIO_FILES_EXTENSIONS); } else { @@ -308,9 +323,7 @@ impl SameMusic { } } - fn check_records_multithreaded(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { - let start_time: SystemTime = SystemTime::now(); - + fn read_tags_load_cache(&mut self) -> (HashMap, HashMap, HashMap) { let loaded_hash_map; let mut records_already_cached: HashMap = Default::default(); @@ -339,148 +352,58 @@ impl SameMusic { loaded_hash_map = Default::default(); mem::swap(&mut self.music_to_check, &mut non_cached_files_to_check); } + (loaded_hash_map, records_already_cached, non_cached_files_to_check) + } + + fn read_tags_save_cache(&mut self, vec_file_entry: Vec, loaded_hash_map: HashMap) { + if !self.use_cache { + return; + } + // Must save all results to file, old loaded from file with all currently counted results + let mut all_results: HashMap = loaded_hash_map; + + for file_entry in vec_file_entry { + all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry); + } + save_cache_to_file(&all_results, &mut self.text_messages, self.save_also_as_json); + } + + fn read_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { + let start_time: SystemTime = SystemTime::now(); + + let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.read_tags_load_cache(); let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - let music_to_check = non_cached_files_to_check.len(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - checking_method: CheckingMethod::None, - current_stage: 1, - max_stage: 2, - entries_checked: atomic_file_counter.load(Ordering::Relaxed), - entries_to_check: music_to_check, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common( + progress_sender, + &progress_thread_run, + &atomic_counter, + 1, + 2, + non_cached_files_to_check.len(), + CheckingMethod::None, + ); // Clean for duplicate files let mut vec_file_entry = non_cached_files_to_check .into_par_iter() - .map(|(path, mut music_entry)| { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); + .map(|(path, music_entry)| { + atomic_counter.fetch_add(1, Ordering::Relaxed); if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { check_was_stopped.store(true, Ordering::Relaxed); return None; } - - let Ok(mut file) = File::open(&path) else{return Some(None)}; - - let result = panic::catch_unwind(move || { - match read_from(&mut file) { - Ok(t) => Some(t), - Err(_inspected) => { - // println!("Failed to open {}", path); - None - } - } - }); - - let tagged_file = if let Ok(t) = result { - match t { - Some(r) => r, - None => { - return Some(Some(music_entry)); - } - } - } else { - let message = create_crash_message("Lofty", &path, "https://github.com/image-rs/image/issues"); - println!("{message}"); - return Some(None); - }; - - let properties = tagged_file.properties(); - - let mut track_title = String::new(); - let mut track_artist = String::new(); - let mut year = String::new(); - let mut genre = String::new(); - - let bitrate = properties.audio_bitrate().unwrap_or(0); - let mut length = properties.duration().as_millis().to_string(); - - if let Some(tag) = tagged_file.primary_tag() { - track_title = tag.get_string(&ItemKey::TrackTitle).unwrap_or("").to_string(); - track_artist = tag.get_string(&ItemKey::TrackArtist).unwrap_or("").to_string(); - year = tag.get_string(&ItemKey::Year).unwrap_or("").to_string(); - genre = tag.get_string(&ItemKey::Genre).unwrap_or("").to_string(); - } - - for tag in tagged_file.tags() { - if track_title.is_empty() { - if let Some(tag_value) = tag.get_string(&ItemKey::TrackTitle) { - track_title = tag_value.to_string(); - } - } - if track_artist.is_empty() { - if let Some(tag_value) = tag.get_string(&ItemKey::TrackArtist) { - track_artist = tag_value.to_string(); - } - } - if year.is_empty() { - if let Some(tag_value) = tag.get_string(&ItemKey::Year) { - year = tag_value.to_string(); - } - } - if genre.is_empty() { - if let Some(tag_value) = tag.get_string(&ItemKey::Genre) { - genre = tag_value.to_string(); - } - } - // println!("{:?}", tag.items()); - } - - if let Ok(old_length_number) = length.parse::() { - let length_number = old_length_number / 60; - let minutes = length_number / 1000; - let seconds = (length_number % 1000) * 6 / 100; - if minutes != 0 || seconds != 0 { - length = format!("{minutes}:{seconds:02}"); - } else if old_length_number > 0 { - // That means, that audio have length smaller that second, but length is properly read - length = "0:01".to_string(); - } else { - length = String::new(); - } - } else { - length = String::new(); - } - - music_entry.track_title = track_title; - music_entry.track_artist = track_artist; - music_entry.year = year; - music_entry.length = length; - music_entry.genre = genre; - music_entry.bitrate = bitrate; - - Some(Some(music_entry)) + Some(self.read_single_file_tag(&path, music_entry)) }) .while_some() .filter(Option::is_some) .map(Option::unwrap) .collect::>(); - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); // Just connect loaded results with already calculated for (_name, file_entry) in records_already_cached { @@ -489,202 +412,177 @@ impl SameMusic { self.music_entries = vec_file_entry.clone(); - if self.use_cache { - // Must save all results to file, old loaded from file with all currently counted results - let mut all_results: HashMap = loaded_hash_map; - - for file_entry in vec_file_entry { - all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry); - } - save_cache_to_file(&all_results, &mut self.text_messages, self.save_also_as_json); - } + self.read_tags_save_cache(vec_file_entry, loaded_hash_map); // Break if stop was clicked after saving to cache if check_was_stopped.load(Ordering::Relaxed) { return false; } - Common::print_time(start_time, SystemTime::now(), "check_records_multithreaded"); + Common::print_time(start_time, SystemTime::now(), "read_tags"); true } - fn check_for_duplicates(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { - assert!(MusicSimilarity::NONE != self.music_similarity, "This can't be none"); + fn read_single_file_tag(&self, path: &str, mut music_entry: MusicEntry) -> Option { + let Ok(mut file) = File::open(path) else { return None; }; + + let result = panic::catch_unwind(move || { + match read_from(&mut file) { + Ok(t) => Some(t), + Err(_inspected) => { + // println!("Failed to open {}", path); + None + } + } + }); + + let tagged_file = if let Ok(t) = result { + match t { + Some(r) => r, + None => { + return Some(music_entry); + } + } + } else { + let message = create_crash_message("Lofty", path, "https://github.com/image-rs/image/issues"); + println!("{message}"); + return None; + }; + + let properties = tagged_file.properties(); + + let mut track_title = String::new(); + let mut track_artist = String::new(); + let mut year = String::new(); + let mut genre = String::new(); + + let bitrate = properties.audio_bitrate().unwrap_or(0); + let mut length = properties.duration().as_millis().to_string(); + + if let Some(tag) = tagged_file.primary_tag() { + track_title = tag.get_string(&ItemKey::TrackTitle).unwrap_or("").to_string(); + track_artist = tag.get_string(&ItemKey::TrackArtist).unwrap_or("").to_string(); + year = tag.get_string(&ItemKey::Year).unwrap_or("").to_string(); + genre = tag.get_string(&ItemKey::Genre).unwrap_or("").to_string(); + } + + for tag in tagged_file.tags() { + if track_title.is_empty() { + if let Some(tag_value) = tag.get_string(&ItemKey::TrackTitle) { + track_title = tag_value.to_string(); + } + } + if track_artist.is_empty() { + if let Some(tag_value) = tag.get_string(&ItemKey::TrackArtist) { + track_artist = tag_value.to_string(); + } + } + if year.is_empty() { + if let Some(tag_value) = tag.get_string(&ItemKey::Year) { + year = tag_value.to_string(); + } + } + if genre.is_empty() { + if let Some(tag_value) = tag.get_string(&ItemKey::Genre) { + genre = tag_value.to_string(); + } + } + // println!("{:?}", tag.items()); + } + + if let Ok(old_length_number) = length.parse::() { + let length_number = old_length_number / 60; + let minutes = length_number / 1000; + let seconds = (length_number % 1000) * 6 / 100; + if minutes != 0 || seconds != 0 { + length = format!("{minutes}:{seconds:02}"); + } else if old_length_number > 0 { + // That means, that audio have length smaller that second, but length is properly read + length = "0:01".to_string(); + } else { + length = String::new(); + } + } else { + length = String::new(); + } + + music_entry.track_title = track_title; + music_entry.track_artist = track_artist; + music_entry.year = year; + music_entry.length = length; + music_entry.genre = genre; + music_entry.bitrate = bitrate; + + Some(music_entry) + } + + fn check_for_duplicate_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { + assert_ne!(MusicSimilarity::NONE, self.music_similarity, "This can't be none"); let start_time: SystemTime = SystemTime::now(); - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - let music_to_check = self.music_to_check.len(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - checking_method: CheckingMethod::None, - current_stage: 2, - max_stage: 2, - entries_checked: atomic_file_counter.load(Ordering::Relaxed), - entries_to_check: music_to_check, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common( + progress_sender, + &progress_thread_run, + &atomic_counter, + 2, + 2, + self.music_to_check.len(), + CheckingMethod::None, + ); let mut old_duplicates: Vec> = vec![self.music_entries.clone()]; let mut new_duplicates: Vec> = Vec::new(); if (self.music_similarity & MusicSimilarity::TRACK_TITLE) == MusicSimilarity::TRACK_TITLE { - for vec_file_entry in old_duplicates { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } - let mut hash_map: BTreeMap> = Default::default(); - for file_entry in vec_file_entry { - let mut thing = file_entry.track_title.trim().to_lowercase(); - if self.approximate_comparison { - get_approximate_conversion(&mut thing); - } - if !thing.is_empty() { - hash_map.entry(thing.clone()).or_insert_with(Vec::new).push(file_entry); - } - } - for (_title, vec_file_entry) in hash_map { - if vec_file_entry.len() > 1 { - new_duplicates.push(vec_file_entry); - } - } + if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); + return false; } - old_duplicates = new_duplicates; - new_duplicates = Vec::new(); + + old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.track_title, self.approximate_comparison); } if (self.music_similarity & MusicSimilarity::TRACK_ARTIST) == MusicSimilarity::TRACK_ARTIST { - for vec_file_entry in old_duplicates { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } - let mut hash_map: BTreeMap> = Default::default(); - for file_entry in vec_file_entry { - let mut thing = file_entry.track_artist.trim().to_lowercase(); - if self.approximate_comparison { - get_approximate_conversion(&mut thing); - } - if !thing.is_empty() { - hash_map.entry(thing.clone()).or_insert_with(Vec::new).push(file_entry); - } - } - for (_title, vec_file_entry) in hash_map { - if vec_file_entry.len() > 1 { - new_duplicates.push(vec_file_entry); - } - } + if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); + return false; } - old_duplicates = new_duplicates; - new_duplicates = Vec::new(); + + old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.track_artist, self.approximate_comparison); } if (self.music_similarity & MusicSimilarity::YEAR) == MusicSimilarity::YEAR { - for vec_file_entry in old_duplicates { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } - let mut hash_map: BTreeMap> = Default::default(); - for file_entry in vec_file_entry { - let thing = file_entry.year.trim().to_lowercase(); - if !thing.is_empty() { - hash_map.entry(thing.clone()).or_insert_with(Vec::new).push(file_entry); - } - } - for (_title, vec_file_entry) in hash_map { - if vec_file_entry.len() > 1 { - new_duplicates.push(vec_file_entry); - } - } + if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); + return false; } - old_duplicates = new_duplicates; - new_duplicates = Vec::new(); + + old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.year, false); } if (self.music_similarity & MusicSimilarity::LENGTH) == MusicSimilarity::LENGTH { - for vec_file_entry in old_duplicates { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } - let mut hash_map: BTreeMap> = Default::default(); - for file_entry in vec_file_entry { - let thing = file_entry.length.trim().to_lowercase(); - if !thing.is_empty() { - hash_map.entry(thing.clone()).or_insert_with(Vec::new).push(file_entry); - } - } - for (_title, vec_file_entry) in hash_map { - if vec_file_entry.len() > 1 { - new_duplicates.push(vec_file_entry); - } - } + if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); + return false; } - old_duplicates = new_duplicates; - new_duplicates = Vec::new(); + + old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.length, false); } if (self.music_similarity & MusicSimilarity::GENRE) == MusicSimilarity::GENRE { - for vec_file_entry in old_duplicates { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } - let mut hash_map: BTreeMap> = Default::default(); - for file_entry in vec_file_entry { - let thing = file_entry.genre.trim().to_lowercase(); - if !thing.is_empty() { - hash_map.entry(thing).or_insert_with(Vec::new).push(file_entry); - } - } - for (_title, vec_file_entry) in hash_map { - if vec_file_entry.len() > 1 { - new_duplicates.push(vec_file_entry); - } - } + if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); + return false; } - old_duplicates = new_duplicates; - new_duplicates = Vec::new(); + + old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.genre, false); } if (self.music_similarity & MusicSimilarity::BITRATE) == MusicSimilarity::BITRATE { + if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); + return false; + } + for vec_file_entry in old_duplicates { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); - return false; - } + atomic_counter.fetch_add(1, Ordering::Relaxed); let mut hash_map: BTreeMap> = Default::default(); for file_entry in vec_file_entry { if file_entry.bitrate != 0 { @@ -701,40 +599,13 @@ impl SameMusic { } } old_duplicates = new_duplicates; - // new_duplicates = Vec::new(); } - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); self.duplicated_music_entries = old_duplicates; - if self.use_reference_folders { - let mut similar_vector = Default::default(); - mem::swap(&mut self.duplicated_music_entries, &mut similar_vector); - let reference_directories = self.directories.reference_directories.clone(); - self.duplicated_music_entries_referenced = similar_vector - .into_iter() - .filter_map(|vec_file_entry| { - let mut files_from_referenced_folders = Vec::new(); - let mut normal_files = Vec::new(); - for file_entry in vec_file_entry { - if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) { - files_from_referenced_folders.push(file_entry); - } else { - normal_files.push(file_entry); - } - } - - if files_from_referenced_folders.is_empty() || normal_files.is_empty() { - None - } else { - Some((files_from_referenced_folders.pop().unwrap(), normal_files)) - } - }) - .collect::)>>(); - } + self.filter_reference_folders(); if self.use_reference_folders { for (_fe, vector) in &self.duplicated_music_entries_referenced { @@ -748,7 +619,7 @@ impl SameMusic { } } - Common::print_time(start_time, SystemTime::now(), "check_for_duplicates"); + Common::print_time(start_time, SystemTime::now(), "check_for_duplicate_tags"); // Clear unused data self.music_entries.clear(); @@ -756,6 +627,66 @@ impl SameMusic { true } + fn check_music_item( + &self, + old_duplicates: Vec>, + atomic_counter: &Arc, + get_item: fn(&MusicEntry) -> &str, + approximate_comparison: bool, + ) -> Vec> { + let mut new_duplicates: Vec<_> = Default::default(); + for vec_file_entry in old_duplicates { + atomic_counter.fetch_add(1, Ordering::Relaxed); + let mut hash_map: BTreeMap> = Default::default(); + for file_entry in vec_file_entry { + let mut thing = get_item(&file_entry).trim().to_lowercase(); + if approximate_comparison { + get_approximate_conversion(&mut thing); + } + if !thing.is_empty() { + hash_map.entry(thing).or_insert_with(Vec::new).push(file_entry); + } + } + for (_title, vec_file_entry) in hash_map { + if vec_file_entry.len() > 1 { + new_duplicates.push(vec_file_entry); + } + } + } + + new_duplicates + } + + fn filter_reference_folders(&mut self) { + if !self.use_reference_folders { + return; + } + + let mut similar_vector = Default::default(); + mem::swap(&mut self.duplicated_music_entries, &mut similar_vector); + let reference_directories = self.directories.reference_directories.clone(); + self.duplicated_music_entries_referenced = similar_vector + .into_iter() + .filter_map(|vec_file_entry| { + let mut files_from_referenced_folders = Vec::new(); + let mut normal_files = Vec::new(); + for file_entry in vec_file_entry { + if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) { + files_from_referenced_folders.push(file_entry); + } else { + normal_files.push(file_entry); + } + } + + if files_from_referenced_folders.is_empty() || normal_files.is_empty() { + None + } else { + Some((files_from_referenced_folders.pop().unwrap(), normal_files)) + } + }) + .collect::)>>(); + } + pub fn set_minimal_file_size(&mut self, minimal_file_size: u64) { self.minimal_file_size = match minimal_file_size { 0 => 1, diff --git a/czkawka_core/src/similar_images.rs b/czkawka_core/src/similar_images.rs index 9f52409..9d0cfba 100644 --- a/czkawka_core/src/similar_images.rs +++ b/czkawka_core/src/similar_images.rs @@ -1,17 +1,17 @@ use std::collections::{BTreeSet, HashMap, HashSet}; -use std::fs::{File, Metadata}; +use std::fs::{DirEntry, File, Metadata}; use std::io::Write; use std::io::*; +use std::mem; use std::panic; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::sleep; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use std::{fs, mem, thread}; +use std::time::SystemTime; use bk_tree::BKTree; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use humansize::format_size; use humansize::BINARY; use image::GenericImageView; @@ -22,16 +22,18 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "heif")] use crate::common::get_dynamic_image_from_heic; use crate::common::{ - create_crash_message, get_dynamic_image_from_raw_image, get_number_of_threads, open_cache_folder, Common, HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, LOOP_DURATION, - RAW_IMAGE_EXTENSIONS, + check_folder_children, create_crash_message, get_dynamic_image_from_raw_image, get_number_of_threads, open_cache_folder, prepare_thread_handler_common, + send_info_and_wait_for_ending_all_threads, Common, HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS, }; +use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; use crate::common_items::ExcludedItems; use crate::common_messages::Messages; use crate::common_traits::{DebugPrint, PrintResults, SaveResults}; use crate::flc; -use crate::localizer_core::generate_translation_hashmap; + +type ImHash = Vec; pub const SIMILAR_VALUES: [[u32; 6]; 4] = [ [1, 2, 5, 7, 14, 20], // 8 @@ -40,21 +42,13 @@ pub const SIMILAR_VALUES: [[u32; 6]; 4] = [ [6, 20, 40, 40, 40, 40], // 64 ]; -#[derive(Debug)] -pub struct ProgressData { - pub current_stage: u8, - pub max_stage: u8, - pub images_checked: usize, - pub images_to_check: usize, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FileEntry { pub path: PathBuf, pub size: u64, pub dimensions: String, pub modified_date: u64, - pub hash: Vec, + pub hash: ImHash, pub similarity: u32, } @@ -74,12 +68,12 @@ pub enum SimilarityPreset { /// Distance metric to use with the BK-tree. struct Hamming; -impl bk_tree::Metric> for Hamming { - fn distance(&self, a: &Vec, b: &Vec) -> u32 { +impl bk_tree::Metric for Hamming { + fn distance(&self, a: &ImHash, b: &ImHash) -> u32 { hamming::distance_fast(a, b).unwrap() as u32 } - fn threshold_distance(&self, a: &Vec, b: &Vec, _threshold: u32) -> Option { + fn threshold_distance(&self, a: &ImHash, b: &ImHash, _threshold: u32) -> Option { Some(self.distance(a, b)) } } @@ -91,13 +85,14 @@ pub struct SimilarImages { directories: Directories, allowed_extensions: Extensions, excluded_items: ExcludedItems, - bktree: BKTree, Hamming>, + bktree: BKTree, similar_vectors: Vec>, similar_referenced_vectors: Vec<(FileEntry, Vec)>, recursive_search: bool, minimal_file_size: u64, maximal_file_size: u64, - image_hashes: HashMap, Vec>, // Hashmap with image hashes and Vector with names of files + image_hashes: HashMap>, + // Hashmap with image hashes and Vector with names of files stopped_search: bool, similarity: u32, images_to_check: HashMap, @@ -252,7 +247,7 @@ impl SimilarImages { } /// Public function used by CLI to search for empty folders - pub fn find_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.directories.optimize_directories(true, &mut self.text_messages); self.use_reference_folders = !self.directories.reference_directories.is_empty(); if !self.check_for_similar_images(stop_receiver, progress_sender) { @@ -279,7 +274,7 @@ impl SimilarImages { /// Function to check if folder are empty. /// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty - fn check_for_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_for_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let start_time: SystemTime = SystemTime::now(); let mut folders_to_check: Vec = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector @@ -301,39 +296,13 @@ impl SimilarImages { folders_to_check.push(id.clone()); } - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 0, - max_stage: 3, - images_checked: atomic_file_counter.load(Ordering::Relaxed), - images_to_check: 0, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 2, 0, CheckingMethod::None); while !folders_to_check.is_empty() { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); return false; } @@ -343,120 +312,29 @@ impl SimilarImages { let mut dir_result = vec![]; let mut warnings = vec![]; let mut fe_result = vec![]; - // Read current dir children - let read_dir = match fs::read_dir(current_folder) { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_open_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - return (dir_result, warnings, fe_result); - } + + let Some(read_dir) = common_read_dir(current_folder, &mut warnings) else { + return (dir_result, warnings, fe_result); }; - // Check every sub folder/file/link etc. - 'dir: for entry in read_dir { - let entry_data = match entry { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_entry_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } - }; - let metadata: Metadata = match entry_data.metadata() { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_metadata_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } + for entry in read_dir { + let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else { + continue; }; + if metadata.is_dir() { - if !self.recursive_search { - continue 'dir; - } - - let next_folder = current_folder.join(entry_data.file_name()); - if self.directories.is_excluded(&next_folder) { - continue 'dir; - } - - if self.excluded_items.is_excluded(&next_folder) { - continue 'dir; - } - - #[cfg(target_family = "unix")] - if self.directories.exclude_other_filesystems() { - match self.directories.is_on_other_filesystems(&next_folder) { - Ok(true) => continue 'dir, - Err(e) => warnings.push(e.to_string()), - _ => (), - } - } - - dir_result.push(next_folder); + check_folder_children( + &mut dir_result, + &mut warnings, + current_folder, + entry_data, + self.recursive_search, + &self.directories, + &self.excluded_items, + ); } else if metadata.is_file() { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - - let file_name_lowercase: String = match entry_data.file_name().into_string() { - Ok(t) => t, - Err(_inspected) => { - warnings.push(flc!( - "core_file_not_utf8_name", - generate_translation_hashmap(vec![("name", entry_data.path().display().to_string())]) - )); - continue 'dir; - } - } - .to_lowercase(); - - if !self.allowed_extensions.matches_filename(&file_name_lowercase) { - continue 'dir; - } - - // Checking files - if (self.minimal_file_size..=self.maximal_file_size).contains(&metadata.len()) { - let current_file_name = current_folder.join(entry_data.file_name()); - if self.excluded_items.is_excluded(¤t_file_name) { - continue 'dir; - } - - let fe: FileEntry = FileEntry { - path: current_file_name.clone(), - size: metadata.len(), - dimensions: String::new(), - modified_date: match metadata.modified() { - Ok(t) => match t.duration_since(UNIX_EPOCH) { - Ok(d) => d.as_secs(), - Err(_inspected) => { - warnings.push(flc!( - "core_file_modified_before_epoch", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string())]) - )); - 0 - } - }, - Err(e) => { - warnings.push(flc!( - "core_file_no_modification_date", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string()), ("reason", e.to_string())]) - )); - 0 - } - }, - - hash: Vec::new(), - similarity: 0, - }; - - fe_result.push((current_file_name.to_string_lossy().to_string(), fe)); - } + atomic_counter.fetch_add(1, Ordering::Relaxed); + self.add_file_entry(&metadata, current_folder, entry_data, &mut fe_result, &mut warnings); } } (dir_result, warnings, fe_result) @@ -476,23 +354,41 @@ impl SimilarImages { } } - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); Common::print_time(start_time, SystemTime::now(), "check_for_similar_images"); true } - // Cache algorithm: - // - Load data from file - // - Remove from data to search, already loaded entries from cache(size and modified datamust match) - // - Check hash of files which doesn't have saved entry - // - Join already read hashes with hashes which were read from file - // - Join all hashes and save it to file + fn add_file_entry(&self, metadata: &Metadata, current_folder: &Path, entry_data: &DirEntry, fe_result: &mut Vec<(String, FileEntry)>, warnings: &mut Vec) { + let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else { + return; + }; - fn hash_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { - let hash_map_modification = SystemTime::now(); + if !self.allowed_extensions.matches_filename(&file_name_lowercase) { + return; + } + // Checking files + if (self.minimal_file_size..=self.maximal_file_size).contains(&metadata.len()) { + let current_file_name = current_folder.join(entry_data.file_name()); + if self.excluded_items.is_excluded(¤t_file_name) { + return; + } + + let fe: FileEntry = FileEntry { + path: current_file_name.clone(), + size: metadata.len(), + dimensions: String::new(), + modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false), + hash: Vec::new(), + similarity: 0, + }; + + fe_result.push((current_file_name.to_string_lossy().to_string(), fe)); + } + } + + fn hash_images_load_cache(&mut self) -> (HashMap, HashMap, HashMap) { let loaded_hash_map; let mut records_already_cached: HashMap = Default::default(); @@ -521,125 +417,53 @@ impl SimilarImages { loaded_hash_map = Default::default(); mem::swap(&mut self.images_to_check, &mut non_cached_files_to_check); } + (loaded_hash_map, records_already_cached, non_cached_files_to_check) + } + + // Cache algorithm: + // - Load data from file + // - Remove from data to search, already loaded entries from cache(size and modified datamust match) + // - Check hash of files which doesn't have saved entry + // - Join already read hashes with hashes which were read from file + // - Join all hashes and save it to file + + fn hash_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { + let hash_map_modification = SystemTime::now(); + + let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.hash_images_load_cache(); Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - reading data from cache and preparing them"); let hash_map_modification = SystemTime::now(); - //// PROGRESS THREAD START let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread let progress_thread_run = Arc::new(AtomicBool::new(true)); + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common( + progress_sender, + &progress_thread_run, + &atomic_counter, + 1, + 2, + non_cached_files_to_check.len(), + CheckingMethod::None, + ); - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - let images_to_check = non_cached_files_to_check.len(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 1, - max_stage: 3, - images_checked: atomic_file_counter.load(Ordering::Relaxed), - images_to_check, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - - //// PROGRESS THREAD END - let mut vec_file_entry: Vec<(FileEntry, Vec)> = non_cached_files_to_check + let mut vec_file_entry: Vec<(FileEntry, ImHash)> = non_cached_files_to_check .into_par_iter() - .map(|(_s, mut file_entry)| { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); + .map(|(_s, file_entry)| { + atomic_counter.fetch_add(1, Ordering::Relaxed); if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { check_was_stopped.store(true, Ordering::Relaxed); return None; } - let file_name_lowercase = file_entry.path.to_string_lossy().to_lowercase(); - - let image; - - #[allow(clippy::never_loop)] // Required to implement nice if/else - 'krztyna: loop { - if RAW_IMAGE_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) { - image = match get_dynamic_image_from_raw_image(&file_entry.path) { - Some(t) => t, - None => return Some(Some((file_entry, Vec::new()))), - }; - break 'krztyna; - } - - #[cfg(feature = "heif")] - if HEIC_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) { - image = match get_dynamic_image_from_heic(&file_entry.path.to_string_lossy()) { - Ok(t) => t, - Err(_) => { - return Some(Some((file_entry, Vec::new()))); - } - }; - break 'krztyna; - } - - // Normal image extension, when any other fail, not using if/else - let result = panic::catch_unwind(|| { - match image::open(file_entry.path.clone()) { - Ok(t) => Ok(t), - // Err(_inspected) => return Some(None), // Something is wrong with image, - // For broken images empty hash is used, because without it will try to resecan files each time when it is called(missing cache file is responsible for it) - // This may cause problems(very rarely), when e.g. file was not available due lack of permissions, but it is available now - Err(_inspected) => Err(()), - } - }); - - // If image crashed during opening, we just skip checking its hash and go on - if let Ok(image_result) = result { - if let Ok(image2) = image_result { - image = image2; - } else { - return Some(Some((file_entry, Vec::new()))); - } - } else { - let message = create_crash_message("Image-rs", &file_entry.path.to_string_lossy(), "https://github.com/image-rs/image/issues"); - println!("{message}"); - return Some(Some((file_entry, Vec::new()))); - } - - break 'krztyna; - } - - let dimensions = image.dimensions(); - - file_entry.dimensions = format!("{}x{}", dimensions.0, dimensions.1); - - let hasher_config = HasherConfig::new() - .hash_size(self.hash_size as u32, self.hash_size as u32) - .hash_alg(self.hash_alg) - .resize_filter(self.image_filter); - let hasher = hasher_config.to_hasher(); - - let hash = hasher.hash_image(&image); - let buf: Vec = hash.as_bytes().to_vec(); - - file_entry.hash = buf.clone(); - - Some(Some((file_entry, buf))) + Some(Some(self.collect_image_file_entry(file_entry))) }) .while_some() .filter(Option::is_some) .map(Option::unwrap) - .collect::)>>(); + .collect::>(); - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - reading data from files in parallel"); let hash_map_modification = SystemTime::now(); @@ -681,8 +505,310 @@ impl SimilarImages { Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - saving data to files"); true } + fn collect_image_file_entry(&self, mut file_entry: FileEntry) -> (FileEntry, ImHash) { + let file_name_lowercase = file_entry.path.to_string_lossy().to_lowercase(); - fn find_similar_hashes(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + let image; + + #[allow(clippy::never_loop)] // Required to implement nice if/else + 'krztyna: loop { + if RAW_IMAGE_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) { + image = match get_dynamic_image_from_raw_image(&file_entry.path) { + Some(t) => t, + None => return (file_entry, Vec::new()), + }; + break 'krztyna; + } + + #[cfg(feature = "heif")] + if HEIC_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) { + image = match get_dynamic_image_from_heic(&file_entry.path.to_string_lossy()) { + Ok(t) => t, + Err(_) => { + return (file_entry, Vec::new()); + } + }; + break 'krztyna; + } + + // Normal image extension, when any other fail, not using if/else + let result = panic::catch_unwind(|| { + match image::open(file_entry.path.clone()) { + Ok(t) => Ok(t), + // Err(_inspected) => return Some(None), // Something is wrong with image, + // For broken images empty hash is used, because without it will try to resecan files each time when it is called(missing cache file is responsible for it) + // This may cause problems(very rarely), when e.g. file was not available due lack of permissions, but it is available now + Err(_inspected) => Err(()), + } + }); + + // If image crashed during opening, we just skip checking its hash and go on + if let Ok(image_result) = result { + if let Ok(image2) = image_result { + image = image2; + } else { + return (file_entry, Vec::new()); + } + } else { + let message = create_crash_message("Image-rs", &file_entry.path.to_string_lossy(), "https://github.com/image-rs/image/issues"); + println!("{message}"); + return (file_entry, Vec::new()); + } + + break 'krztyna; + } + + let dimensions = image.dimensions(); + + file_entry.dimensions = format!("{}x{}", dimensions.0, dimensions.1); + + let hasher_config = HasherConfig::new() + .hash_size(self.hash_size as u32, self.hash_size as u32) + .hash_alg(self.hash_alg) + .resize_filter(self.image_filter); + let hasher = hasher_config.to_hasher(); + + let hash = hasher.hash_image(&image); + let buf: ImHash = hash.as_bytes().to_vec(); + + file_entry.hash = buf.clone(); + + (file_entry, buf) + } + + fn compare_hashes( + &self, + hashes_to_check: &[ImHash], + atomic_counter: &Arc, + stop_receiver: Option<&Receiver<()>>, + check_was_stopped: &AtomicBool, + tolerance: u32, + hashes_with_multiple_images: &HashSet, + all_hashed_images: &HashMap>, + ) -> Option<(HashMap, HashMap)> { + let mut hashes_parents: HashMap = Default::default(); // Hashes used as parent (hash, children_number_of_hash) + let mut hashes_similarity: HashMap = Default::default(); // Hashes used as child, (parent_hash, similarity) + + // Sprawdź czy hash nie jest użyty jako master gdzie indziej + // JeÅ›li tak to przejdź do sprawdzania kolejnego elementu + // Zweryfikuj czy sprawdzany element ma rodzica + // JeÅ›li ma to sprawdź czy similarity nowego rodzica jest mniejsze niż starego + // // JeÅ›li tak to zmniejsz ilość dzieci starego rodzica, dodaj ilość dzieci w nowym rodzicu i podmieÅ„ rekord hashes_similarity + // // JeÅ›li nie to dodaj nowy rekord w hashes_similarity jak i hashes_parents z liczbÄ… dzieci równÄ… 1 + + for (index, hash_to_check) in hashes_to_check.iter().enumerate() { + // Don't check for user stop too often + // Also don't add too often data to atomic variable + const CYCLES_COUNTER: usize = 0b11_1111; + if ((index & CYCLES_COUNTER) == CYCLES_COUNTER) && index != 0 { + atomic_counter.fetch_add(CYCLES_COUNTER, Ordering::Relaxed); + if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { + check_was_stopped.store(true, Ordering::Relaxed); + return None; + } + } + hashes_parents.insert((*hash_to_check).clone(), 0); + + let mut found_items = self + .bktree + .find(hash_to_check, tolerance) + .filter(|(similarity, _hash)| if self.use_reference_folders { true } else { *similarity != 0 }) + .collect::>(); + + found_items.sort_unstable_by_key(|f| f.0); + + for (similarity, compared_hash) in found_items { + image_to_check( + &mut hashes_parents, + &mut hashes_similarity, + hashes_with_multiple_images, + hash_to_check, + compared_hash, + similarity, + ); + } + } + + debug_check_for_duplicated_things(self.use_reference_folders, &hashes_parents, &hashes_similarity, all_hashed_images, "BEFORE"); + + Some((hashes_parents, hashes_similarity)) + } + + fn chunk_hashes(&mut self, all_hashed_images: &HashMap>, all_hashes: &Vec) -> (Vec>, HashSet) { + let mut hashes_with_multiple_images: HashSet = Default::default(); // Fast way to check if hash have multiple images + let mut files_from_referenced_folders: HashMap> = HashMap::new(); + let mut normal_files: HashMap> = HashMap::new(); + + let number_of_processors = get_number_of_threads(); + let chunk_size; + + let mut initial_hashes: Vec = Vec::new(); + let mut additional_chunk_to_check: Vec = Default::default(); + + let mut chunks: Vec>; + if self.use_reference_folders { + let reference_directories = self.directories.reference_directories.clone(); + all_hashed_images.clone().into_iter().for_each(|(hash, vec_file_entry)| { + for file_entry in vec_file_entry { + if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) { + files_from_referenced_folders.entry(hash.clone()).or_insert_with(Vec::new).push(file_entry); + } else { + normal_files.entry(hash.clone()).or_insert_with(Vec::new).push(file_entry); + } + } + }); + + for (hash, vec_files) in normal_files { + if vec_files.len() >= 2 { + hashes_with_multiple_images.insert(hash.clone()); + } + self.bktree.add(hash); + } + for (hash, vec_files) in files_from_referenced_folders { + if vec_files.len() >= 2 { + hashes_with_multiple_images.insert(hash.clone()); + } + initial_hashes.push(hash); + } + chunk_size = initial_hashes.len() / number_of_processors; + + chunks = if chunk_size > 0 { + initial_hashes.chunks(chunk_size).map(<[std::vec::Vec]>::to_vec).collect::>() + } else { + vec![initial_hashes] + }; + } else { + for (hash, vec_files) in all_hashed_images { + if vec_files.len() >= 2 { + additional_chunk_to_check.push(hash.clone()); + hashes_with_multiple_images.insert(hash.clone()); + } else { + self.bktree.add(hash.clone()); + } + } + chunk_size = all_hashes.len() / number_of_processors; + chunks = if chunk_size > 0 { + all_hashes.chunks(chunk_size).map(<[Vec]>::to_vec).collect::>() + } else { + vec![all_hashes.clone()] + }; + chunks.push(additional_chunk_to_check); + } + + (chunks, hashes_with_multiple_images) + } + + fn collect_hash_compare_result( + &self, + hashes_parents: HashMap, + hashes_with_multiple_images: &HashSet, + all_hashed_images: &HashMap>, + collected_similar_images: &mut HashMap>, + hashes_similarity: HashMap, + ) { + if self.use_reference_folders { + // This is same step as without reference folders, but also checks if children are inside/outside reference directories, because may happen, that one file is inside reference folder and other outside + + // Collecting results to vector + for (parent_hash, child_number) in hashes_parents { + // If hash contains other hasher OR multiple images are available for checked hash + if child_number > 0 || hashes_with_multiple_images.contains(&parent_hash) { + let vec_fe = all_hashed_images + .get(&parent_hash) + .unwrap() + .iter() + .filter(|e| is_in_reference_folder(&self.directories.reference_directories, &e.path)) + .cloned() + .collect(); + collected_similar_images.insert(parent_hash.clone(), vec_fe); + } + } + + for (child_hash, (parent_hash, similarity)) in hashes_similarity { + let mut vec_fe: Vec<_> = all_hashed_images + .get(&child_hash) + .unwrap() + .iter() + .filter(|e| !is_in_reference_folder(&self.directories.reference_directories, &e.path)) + .cloned() + .collect(); + for mut fe in &mut vec_fe { + fe.similarity = similarity; + } + collected_similar_images.get_mut(&parent_hash).unwrap().append(&mut vec_fe); + } + } else { + // Collecting results to vector + for (parent_hash, child_number) in hashes_parents { + // If hash contains other hasher OR multiple images are available for checked hash + if child_number > 0 || hashes_with_multiple_images.contains(&parent_hash) { + let vec_fe = all_hashed_images.get(&parent_hash).unwrap().clone(); + collected_similar_images.insert(parent_hash.clone(), vec_fe); + } + } + + for (child_hash, (parent_hash, similarity)) in hashes_similarity { + let mut vec_fe = all_hashed_images.get(&child_hash).unwrap().clone(); + for mut fe in &mut vec_fe { + fe.similarity = similarity; + } + collected_similar_images.get_mut(&parent_hash).unwrap().append(&mut vec_fe); + } + } + } + + fn check_for_duplicate_hashes( + &self, + parts: Vec<(HashMap, HashMap)>, + hashes_with_multiple_images: &HashSet, + all_hashed_images: &HashMap>, + collected_similar_images: &mut HashMap>, + ) { + let mut hashes_parents: HashMap = Default::default(); + let mut hashes_similarity: HashMap = Default::default(); + let mut iter = parts.into_iter(); + // At start fill arrays with first item + // Normal algorithm would do exactly same thing, but slower, one record after one + if let Some((first_hashes_parents, first_hashes_similarity)) = iter.next() { + hashes_parents = first_hashes_parents; + hashes_similarity = first_hashes_similarity; + } + + for (partial_hashes_with_parents, partial_hashes_with_similarity) in iter { + for (parent_hash, _child_number) in partial_hashes_with_parents { + if !hashes_parents.contains_key(&parent_hash) && !hashes_similarity.contains_key(&parent_hash) { + hashes_parents.insert(parent_hash, 0); + } + } + + for (hash_to_check, (compared_hash, similarity)) in partial_hashes_with_similarity { + image_to_check( + &mut hashes_parents, + &mut hashes_similarity, + hashes_with_multiple_images, + &hash_to_check, + &compared_hash, + similarity, + ); + } + } + + debug_check_for_duplicated_things(self.use_reference_folders, &hashes_parents, &hashes_similarity, all_hashed_images, "LATTER"); + + // Just simple check if all original hashes with multiple entries are available in end results + let original_hashes_at_start = hashes_with_multiple_images.len(); + let original_hashes_in_end_results = hashes_parents + .iter() + .filter(|(parent_hash, _child_number)| hashes_with_multiple_images.contains(*parent_hash)) + .count(); + if !self.use_reference_folders { + assert_eq!(original_hashes_at_start, original_hashes_in_end_results); + } + + self.collect_hash_compare_result(hashes_parents, hashes_with_multiple_images, all_hashed_images, collected_similar_images, hashes_similarity); + } + + fn find_similar_hashes(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { if self.image_hashes.is_empty() { return true; } @@ -691,12 +817,12 @@ impl SimilarImages { let tolerance = self.similarity; // Results - let mut collected_similar_images: HashMap, Vec> = Default::default(); + let mut collected_similar_images: HashMap> = Default::default(); let mut all_hashed_images = Default::default(); mem::swap(&mut all_hashed_images, &mut self.image_hashes); - let all_hashes: Vec<_> = all_hashed_images.keys().collect(); + let all_hashes: Vec<_> = all_hashed_images.clone().into_keys().collect(); // Checking entries with tolerance 0 is really easy and fast, because only entries with same hashes needs to be checked if tolerance == 0 { @@ -706,288 +832,71 @@ impl SimilarImages { } } } else { - //// PROGRESS THREAD START let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread let progress_thread_run = Arc::new(AtomicBool::new(true)); - let atomic_mode_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_mode_counter = atomic_mode_counter.clone(); - let all_combinations_to_check = all_hashes.len(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 2, - max_stage: 2, - images_checked: atomic_mode_counter.load(Ordering::Relaxed), - images_to_check: all_combinations_to_check, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 2, 2, all_hashes.len(), CheckingMethod::None); // Don't use hashes with multiple images in bktree, because they will always be master of group and cannot be find by other hashes - let mut hashes_with_multiple_images: HashSet<_> = Default::default(); // Fast way to check if hash have multiple images - let mut files_from_referenced_folders = HashMap::new(); - let mut normal_files = HashMap::new(); - - let number_of_processors = get_number_of_threads(); - let chunk_size; - let mut chunks: Vec<&[&Vec]>; - - let mut initial_hashes: Vec<&Vec> = Vec::new(); - let mut additional_chunk_to_check: Vec<&Vec> = Default::default(); - - if self.use_reference_folders { - let reference_directories = self.directories.reference_directories.clone(); - all_hashed_images.clone().into_iter().for_each(|(hash, vec_file_entry)| { - for file_entry in vec_file_entry { - if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) { - files_from_referenced_folders.entry(hash.clone()).or_insert_with(Vec::new).push(file_entry); - } else { - normal_files.entry(hash.clone()).or_insert_with(Vec::new).push(file_entry); - } - } - }); - - for (hash, vec_files) in &normal_files { - if vec_files.len() >= 2 { - hashes_with_multiple_images.insert(hash); - } - self.bktree.add(hash.clone()); - } - for (hash, vec_files) in &files_from_referenced_folders { - if vec_files.len() >= 2 { - hashes_with_multiple_images.insert(hash); - } - initial_hashes.push(hash); - } - chunk_size = initial_hashes.len() / number_of_processors; - - chunks = if chunk_size > 0 { - initial_hashes.chunks(chunk_size).collect::>() - } else { - vec![&initial_hashes] - }; - } else { - for (hash, vec_files) in &all_hashed_images { - if vec_files.len() >= 2 { - additional_chunk_to_check.push(hash); - hashes_with_multiple_images.insert(hash); - } else { - self.bktree.add(hash.clone()); - } - } - chunk_size = all_hashes.len() / number_of_processors; - chunks = if chunk_size > 0 { - all_hashes.chunks(chunk_size).collect::>() - } else { - vec![&all_hashes] - }; - chunks.push(&additional_chunk_to_check); - } + let (chunks, hashes_with_multiple_images) = self.chunk_hashes(&all_hashed_images, &all_hashes); let parts: Vec<_> = chunks .into_par_iter() .map(|hashes_to_check| { - let mut hashes_parents: HashMap<&Vec, u32> = Default::default(); // Hashes used as parent (hash, children_number_of_hash) - let mut hashes_similarity: HashMap<&Vec, (&Vec, u32)> = Default::default(); // Hashes used as child, (parent_hash, similarity) - - // Sprawdź czy hash nie jest użyty jako master gdzie indziej - // JeÅ›li tak to przejdź do sprawdzania kolejnego elementu - // Zweryfikuj czy sprawdzany element ma rodzica - // JeÅ›li ma to sprawdź czy similarity nowego rodzica jest mniejsze niż starego - // // JeÅ›li tak to zmniejsz ilość dzieci starego rodzica, dodaj ilość dzieci w nowym rodzicu i podmieÅ„ rekord hashes_similarity - // // JeÅ›li nie to dodaj nowy rekord w hashes_similarity jak i hashes_parents z liczbÄ… dzieci równÄ… 1 - - for (index, hash_to_check) in hashes_to_check.iter().enumerate() { - // Don't check for user stop too often - // Also don't add too often data to atomic variable - const CYCLES_COUNTER: usize = 0b11_1111; - if ((index & CYCLES_COUNTER) == CYCLES_COUNTER) && index != 0 { - atomic_mode_counter.fetch_add(CYCLES_COUNTER, Ordering::Relaxed); - if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - check_was_stopped.store(true, Ordering::Relaxed); - return None; - } - } - hashes_parents.insert(hash_to_check, 0); - - let mut found_items = self - .bktree - .find(hash_to_check, tolerance) - .filter(|(similarity, _hash)| if self.use_reference_folders { true } else { *similarity != 0 }) - .collect::>(); - - found_items.sort_unstable_by_key(|f| f.0); - - for (similarity, compared_hash) in found_items { - image_to_check( - &mut hashes_parents, - &mut hashes_similarity, - &hashes_with_multiple_images, - hash_to_check, - compared_hash, - similarity, - ); - } - } - - #[cfg(debug_assertions)] - if !self.use_reference_folders { - debug_check_for_duplicated_things(&hashes_parents, &hashes_similarity, &all_hashed_images, "BEFORE"); - } - - Some((hashes_parents, hashes_similarity)) + self.compare_hashes( + &hashes_to_check, + &atomic_counter, + stop_receiver, + &check_was_stopped, + tolerance, + &hashes_with_multiple_images, + &all_hashed_images, + ) }) .while_some() .collect(); - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); if check_was_stopped.load(Ordering::Relaxed) { return false; } - { - let mut hashes_parents: HashMap<&Vec, u32> = Default::default(); - let mut hashes_similarity: HashMap<&Vec, (&Vec, u32)> = Default::default(); - let mut iter = parts.into_iter(); - // At start fill arrays with first item - // Normal algorithm would do exactly same thing, but slower, one record after one - if let Some((first_hashes_parents, first_hashes_similarity)) = iter.next() { - hashes_parents = first_hashes_parents; - hashes_similarity = first_hashes_similarity; - } - - for (partial_hashes_with_parents, partial_hashes_with_similarity) in iter { - for (parent_hash, _child_number) in partial_hashes_with_parents { - if !hashes_parents.contains_key(parent_hash) && !hashes_similarity.contains_key(parent_hash) { - hashes_parents.insert(parent_hash, 0); - } - } - - for (hash_to_check, (compared_hash, similarity)) in partial_hashes_with_similarity { - image_to_check( - &mut hashes_parents, - &mut hashes_similarity, - &hashes_with_multiple_images, - hash_to_check, - compared_hash, - similarity, - ); - } - } - - #[cfg(debug_assertions)] - if !self.use_reference_folders { - debug_check_for_duplicated_things(&hashes_parents, &hashes_similarity, &all_hashed_images, "LATTER"); - } - - // Just simple check if all original hashes with multiple entries are available in end results - let original_hashes_at_start = hashes_with_multiple_images.len(); - let original_hashes_in_end_results = hashes_parents - .iter() - .filter(|(parent_hash, _child_number)| hashes_with_multiple_images.contains(*parent_hash)) - .count(); - if !self.use_reference_folders { - assert_eq!(original_hashes_at_start, original_hashes_in_end_results); - } - - if self.use_reference_folders { - // This is same step as without reference folders, but also checks if children are inside/outside reference directories, because may happen, that one file is inside reference folder and other outside - - // Collecting results to vector - for (parent_hash, child_number) in hashes_parents { - // If hash contains other hasher OR multiple images are available for checked hash - if child_number > 0 || hashes_with_multiple_images.contains(parent_hash) { - let vec_fe = all_hashed_images - .get(parent_hash) - .unwrap() - .iter() - .filter(|e| is_in_reference_folder(&self.directories.reference_directories, &e.path)) - .cloned() - .collect(); - collected_similar_images.insert(parent_hash.clone(), vec_fe); - } - } - - for (child_hash, (parent_hash, similarity)) in hashes_similarity { - let mut vec_fe: Vec<_> = all_hashed_images - .get(child_hash) - .unwrap() - .iter() - .filter(|e| !is_in_reference_folder(&self.directories.reference_directories, &e.path)) - .cloned() - .collect(); - for mut fe in &mut vec_fe { - fe.similarity = similarity; - } - collected_similar_images.get_mut(parent_hash).unwrap().append(&mut vec_fe); - } - } else { - // Collecting results to vector - for (parent_hash, child_number) in hashes_parents { - // If hash contains other hasher OR multiple images are available for checked hash - if child_number > 0 || hashes_with_multiple_images.contains(parent_hash) { - let vec_fe = all_hashed_images.get(parent_hash).unwrap().clone(); - collected_similar_images.insert(parent_hash.clone(), vec_fe); - } - } - - for (child_hash, (parent_hash, similarity)) in hashes_similarity { - let mut vec_fe = all_hashed_images.get(child_hash).unwrap().clone(); - for mut fe in &mut vec_fe { - fe.similarity = similarity; - } - collected_similar_images.get_mut(parent_hash).unwrap().append(&mut vec_fe); - } - } - } + self.check_for_duplicate_hashes(parts, &hashes_with_multiple_images, &all_hashed_images, &mut collected_similar_images); } - // Validating if group contains duplicated results - #[cfg(debug_assertions)] - { - let mut result_hashset: HashSet = Default::default(); - let mut found = false; - for vec_file_entry in collected_similar_images.values() { - if vec_file_entry.is_empty() { - println!("Empty group"); - found = true; - continue; - } - if vec_file_entry.len() == 1 { - println!("Single Element {vec_file_entry:?}"); - found = true; - continue; - } - for file_entry in vec_file_entry { - let st = file_entry.path.to_string_lossy().to_string(); - if result_hashset.contains(&st) { - found = true; - println!("Duplicated Element {st}"); - } else { - result_hashset.insert(st); - } - } - } - assert!(!found, "Found Invalid entries, verify errors before"); // TODO crashes with empty result with reference folder, verify why - } + self.verify_duplicated_items(&collected_similar_images); + self.similar_vectors = collected_similar_images.into_values().collect(); + self.exclude_items_with_same_size(); + + self.check_for_reference_folders(); + + Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - selecting data from HashMap"); + + if self.use_reference_folders { + for (_fe, vector) in &self.similar_referenced_vectors { + self.information.number_of_duplicates += vector.len(); + self.information.number_of_groups += 1; + } + } else { + for vector in &self.similar_vectors { + self.information.number_of_duplicates += vector.len() - 1; + self.information.number_of_groups += 1; + } + } + + // Clean unused data + self.image_hashes = Default::default(); + self.images_to_check = Default::default(); + self.bktree = BKTree::new(Hamming); + + true + } + + fn exclude_items_with_same_size(&mut self) { if self.exclude_images_with_same_size { let mut new_vector = Default::default(); mem::swap(&mut self.similar_vectors, &mut new_vector); @@ -1005,7 +914,9 @@ impl SimilarImages { } } } + } + fn check_for_reference_folders(&mut self) { if self.use_reference_folders { let mut similar_vector = Default::default(); mem::swap(&mut self.similar_vectors, &mut similar_vector); @@ -1031,27 +942,39 @@ impl SimilarImages { }) .collect::)>>(); } + } - Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - selecting data from HashMap"); - - if self.use_reference_folders { - for (_fe, vector) in &self.similar_referenced_vectors { - self.information.number_of_duplicates += vector.len(); - self.information.number_of_groups += 1; + #[allow(dead_code)] + #[allow(unreachable_code)] + #[allow(unused_variables)] + pub fn verify_duplicated_items(&self, collected_similar_images: &HashMap>) { + #[cfg(not(debug_assertions))] + return; + // Validating if group contains duplicated results + let mut result_hashset: HashSet = Default::default(); + let mut found = false; + for vec_file_entry in collected_similar_images.values() { + if vec_file_entry.is_empty() { + println!("Empty group"); + found = true; + continue; } - } else { - for vector in &self.similar_vectors { - self.information.number_of_duplicates += vector.len() - 1; - self.information.number_of_groups += 1; + if vec_file_entry.len() == 1 { + println!("Single Element {vec_file_entry:?}"); + found = true; + continue; + } + for file_entry in vec_file_entry { + let st = file_entry.path.to_string_lossy().to_string(); + if result_hashset.contains(&st) { + found = true; + println!("Duplicated Element {st}"); + } else { + result_hashset.insert(st); + } } } - - // Clean unused data - self.image_hashes = Default::default(); - self.images_to_check = Default::default(); - self.bktree = BKTree::new(Hamming); - - true + assert!(!found, "Found Invalid entries, verify errors before"); // TODO crashes with empty result with reference folder, verify why } /// Set included dir which needs to be relative, exists etc. @@ -1073,11 +996,11 @@ impl SimilarImages { } fn image_to_check<'a>( - hashes_parents: &mut HashMap<&'a Vec, u32>, - hashes_similarity: &mut HashMap<&'a Vec, (&'a Vec, u32)>, - hashes_with_multiple_images: &HashSet<&'a Vec>, - hash_to_check: &'a Vec, - compared_hash: &'a Vec, + hashes_parents: &mut HashMap, + hashes_similarity: &mut HashMap, + hashes_with_multiple_images: &HashSet, + hash_to_check: &'a ImHash, + compared_hash: &'a ImHash, similarity: u32, ) { if let Some(children_number) = hashes_parents.get(compared_hash) { @@ -1117,12 +1040,12 @@ fn image_to_check<'a>( } if need_to_add { - hashes_similarity.insert(compared_hash, (hash_to_check, similarity)); + hashes_similarity.insert(compared_hash.clone(), (hash_to_check.clone(), similarity)); if let Some(number_of_children) = hashes_parents.get_mut(hash_to_check) { *number_of_children += 1; } else { - hashes_parents.insert(hash_to_check, 1); + hashes_parents.insert(hash_to_check.clone(), 1); } } } @@ -1328,26 +1251,6 @@ pub fn get_string_from_similarity(similarity: &u32, hash_size: u8) -> String { _ => panic!(), }; - // #[cfg(debug_assertions)] - // { - // if *similarity <= SIMILAR_VALUES[index_preset][0] { - // format!("{} {}", flc!("core_similarity_very_high"), *similarity) - // } else if *similarity <= SIMILAR_VALUES[index_preset][1] { - // format!("{} {}", flc!("core_similarity_high"), *similarity) - // } else if *similarity <= SIMILAR_VALUES[index_preset][2] { - // format!("{} {}", flc!("core_similarity_medium"), *similarity) - // } else if *similarity <= SIMILAR_VALUES[index_preset][3] { - // format!("{} {}", flc!("core_similarity_small"), *similarity) - // } else if *similarity <= SIMILAR_VALUES[index_preset][4] { - // format!("{} {}", flc!("core_similarity_very_small"), *similarity) - // } else if *similarity <= SIMILAR_VALUES[index_preset][5] { - // format!("{} {}", flc!("core_similarity_minimal"), *similarity) - // } else { - // panic!(); - // } - // } - // #[cfg(not(debug_assertions))] - if *similarity == 0 { flc!("core_similarity_original") } else if *similarity <= SIMILAR_VALUES[index_preset][0] { @@ -1452,27 +1355,37 @@ pub fn test_image_conversion_speed() { } #[allow(dead_code)] +#[allow(unreachable_code)] +#[allow(unused_variables)] // Function to validate if after first check there are any duplicated entries // E.g. /a.jpg is used also as master and similar image which is forbidden, because may // cause accidentally delete more pictures that user wanted fn debug_check_for_duplicated_things( - hashes_parents: &HashMap<&Vec, u32>, - hashes_similarity: &HashMap<&Vec, (&Vec, u32)>, - all_hashed_images: &HashMap, Vec>, + use_reference_folders: bool, + hashes_parents: &HashMap, + hashes_similarity: &HashMap, + all_hashed_images: &HashMap>, numm: &str, ) { + #[cfg(not(debug_assertions))] + return; + + if use_reference_folders { + return; + } + let mut found_broken_thing = false; let mut hashmap_hashes: HashSet<_> = Default::default(); let mut hashmap_names: HashSet<_> = Default::default(); for (hash, number_of_children) in hashes_parents { if *number_of_children > 0 { - if hashmap_hashes.contains(*hash) { - println!("------1--HASH--{} {:?}", numm, all_hashed_images.get(*hash).unwrap()); + if hashmap_hashes.contains(hash) { + println!("------1--HASH--{} {:?}", numm, all_hashed_images.get(hash).unwrap()); found_broken_thing = true; } hashmap_hashes.insert((*hash).clone()); - for i in all_hashed_images.get(*hash).unwrap() { + for i in all_hashed_images.get(hash).unwrap() { let name = i.path.to_string_lossy().to_string(); if hashmap_names.contains(&name) { println!("------1--NAME--{numm} {name:?}"); @@ -1483,13 +1396,13 @@ fn debug_check_for_duplicated_things( } } for hash in hashes_similarity.keys() { - if hashmap_hashes.contains(*hash) { - println!("------2--HASH--{} {:?}", numm, all_hashed_images.get(*hash).unwrap()); + if hashmap_hashes.contains(hash) { + println!("------2--HASH--{} {:?}", numm, all_hashed_images.get(hash).unwrap()); found_broken_thing = true; } hashmap_hashes.insert((*hash).clone()); - for i in all_hashed_images.get(*hash).unwrap() { + for i in all_hashed_images.get(hash).unwrap() { let name = i.path.to_string_lossy().to_string(); if hashmap_names.contains(&name) { println!("------2--NAME--{numm} {name:?}"); @@ -1499,7 +1412,5 @@ fn debug_check_for_duplicated_things( } } - if found_broken_thing { - panic!(); - } + assert!(!found_broken_thing); } diff --git a/czkawka_core/src/similar_videos.rs b/czkawka_core/src/similar_videos.rs index 947d781..aaba58c 100644 --- a/czkawka_core/src/similar_videos.rs +++ b/czkawka_core/src/similar_videos.rs @@ -1,16 +1,16 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::fs::{File, Metadata}; +use std::fs::{DirEntry, File, Metadata}; use std::io::Write; use std::io::*; +use std::mem; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::sleep; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use std::{fs, mem, thread}; +use std::time::SystemTime; use crossbeam_channel::Receiver; use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound; +use futures::channel::mpsc::UnboundedSender; use humansize::format_size; use humansize::BINARY; use rayon::prelude::*; @@ -18,8 +18,9 @@ use serde::{Deserialize, Serialize}; use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo; use vid_dup_finder_lib::{NormalizedTolerance, VideoHash}; -use crate::common::VIDEO_FILES_EXTENSIONS; -use crate::common::{open_cache_folder, Common, LOOP_DURATION}; +use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS}; +use crate::common::{open_cache_folder, Common}; +use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData}; use crate::common_directory::Directories; use crate::common_extensions::Extensions; use crate::common_items::ExcludedItems; @@ -30,14 +31,6 @@ use crate::localizer_core::generate_translation_hashmap; pub const MAX_TOLERANCE: i32 = 20; -#[derive(Debug)] -pub struct ProgressData { - pub current_stage: u8, - pub max_stage: u8, - pub videos_checked: usize, - pub videos_to_check: usize, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FileEntry { pub path: PathBuf, @@ -214,7 +207,7 @@ impl SimilarVideos { } /// Public function used by CLI to search for empty folders - pub fn find_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { if !check_if_ffmpeg_is_installed() { self.text_messages.errors.push(flc!("core_ffmpeg_not_found")); #[cfg(target_os = "windows")] @@ -248,7 +241,7 @@ impl SimilarVideos { /// Function to check if folder are empty. /// Parameter `initial_checking` for second check before deleting to be sure that checked folder is still empty - fn check_for_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_for_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let start_time: SystemTime = SystemTime::now(); let mut folders_to_check: Vec = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector @@ -266,39 +259,13 @@ impl SimilarVideos { folders_to_check.push(id.clone()); } - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 0, - max_stage: 1, - videos_checked: atomic_file_counter.load(Ordering::Relaxed), - videos_to_check: 0, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 1, 0, CheckingMethod::None); while !folders_to_check.is_empty() { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); return false; } @@ -308,119 +275,30 @@ impl SimilarVideos { let mut dir_result = vec![]; let mut warnings = vec![]; let mut fe_result = vec![]; - // Read current dir children - let read_dir = match fs::read_dir(current_folder) { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_open_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - return (dir_result, warnings, fe_result); - } + + let Some(read_dir) = common_read_dir(current_folder, &mut warnings) else { + return (dir_result, warnings, fe_result); }; // Check every sub folder/file/link etc. - 'dir: for entry in read_dir { - let entry_data = match entry { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_entry_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } - }; - let metadata: Metadata = match entry_data.metadata() { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_metadata_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } + for entry in read_dir { + let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else { + continue; }; + if metadata.is_dir() { - if !self.recursive_search { - continue 'dir; - } - - let next_folder = current_folder.join(entry_data.file_name()); - if self.directories.is_excluded(&next_folder) { - continue 'dir; - } - - if self.excluded_items.is_excluded(&next_folder) { - continue 'dir; - } - - #[cfg(target_family = "unix")] - if self.directories.exclude_other_filesystems() { - match self.directories.is_on_other_filesystems(&next_folder) { - Ok(true) => continue 'dir, - Err(e) => warnings.push(e.to_string()), - _ => (), - } - } - - dir_result.push(next_folder); + check_folder_children( + &mut dir_result, + &mut warnings, + current_folder, + entry_data, + self.recursive_search, + &self.directories, + &self.excluded_items, + ); } else if metadata.is_file() { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - - let file_name_lowercase: String = match entry_data.file_name().into_string() { - Ok(t) => t, - Err(_inspected) => { - warnings.push(flc!( - "core_file_not_utf8_name", - generate_translation_hashmap(vec![("name", entry_data.path().display().to_string())]) - )); - continue 'dir; - } - } - .to_lowercase(); - - if !self.allowed_extensions.matches_filename(&file_name_lowercase) { - continue 'dir; - } - - // Checking files - if (self.minimal_file_size..=self.maximal_file_size).contains(&metadata.len()) { - let current_file_name = current_folder.join(entry_data.file_name()); - if self.excluded_items.is_excluded(¤t_file_name) { - continue 'dir; - } - let current_file_name_str = current_file_name.to_string_lossy().to_string(); - - let fe: FileEntry = FileEntry { - path: current_file_name.clone(), - size: metadata.len(), - modified_date: match metadata.modified() { - Ok(t) => match t.duration_since(UNIX_EPOCH) { - Ok(d) => d.as_secs(), - Err(_inspected) => { - warnings.push(flc!( - "core_file_modified_before_epoch", - generate_translation_hashmap(vec![("name", current_file_name_str.clone())]) - )); - 0 - } - }, - Err(e) => { - warnings.push(flc!( - "core_file_no_modification_date", - generate_translation_hashmap(vec![("name", current_file_name_str.clone()), ("reason", e.to_string())]) - )); - 0 - } - }, - vhash: Default::default(), - error: String::new(), - }; - - fe_result.push((current_file_name_str, fe)); - } + atomic_counter.fetch_add(1, Ordering::Relaxed); + self.add_video_file_entry(&metadata, entry_data, &mut fe_result, &mut warnings, current_folder); } } (dir_result, warnings, fe_result) @@ -440,18 +318,43 @@ impl SimilarVideos { } } - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); Common::print_time(start_time, SystemTime::now(), "check_for_similar_videos"); true } - fn sort_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { - let hash_map_modification = SystemTime::now(); + fn add_video_file_entry(&self, metadata: &Metadata, entry_data: &DirEntry, fe_result: &mut Vec<(String, FileEntry)>, warnings: &mut Vec, current_folder: &Path) { + let Some(file_name_lowercase) = get_lowercase_name(entry_data, + warnings) else { + return; + }; + if !self.allowed_extensions.matches_filename(&file_name_lowercase) { + return; + } + + // Checking files + if (self.minimal_file_size..=self.maximal_file_size).contains(&metadata.len()) { + let current_file_name = current_folder.join(entry_data.file_name()); + if self.excluded_items.is_excluded(¤t_file_name) { + return; + } + let current_file_name_str = current_file_name.to_string_lossy().to_string(); + + let fe: FileEntry = FileEntry { + path: current_file_name.clone(), + size: metadata.len(), + modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false), + vhash: Default::default(), + error: String::new(), + }; + + fe_result.push((current_file_name_str, fe)); + } + } + + fn load_cache_at_start(&mut self) -> (BTreeMap, BTreeMap, BTreeMap) { let loaded_hash_map; - let mut records_already_cached: BTreeMap = Default::default(); let mut non_cached_files_to_check: BTreeMap = Default::default(); @@ -478,43 +381,35 @@ impl SimilarVideos { loaded_hash_map = Default::default(); mem::swap(&mut self.videos_to_check, &mut non_cached_files_to_check); } + (loaded_hash_map, records_already_cached, non_cached_files_to_check) + } + + fn sort_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { + let hash_map_modification = SystemTime::now(); + + let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.load_cache_at_start(); Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - reading data from cache and preparing them"); let hash_map_modification = SystemTime::now(); - //// PROGRESS THREAD START let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread let progress_thread_run = Arc::new(AtomicBool::new(true)); - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common( + progress_sender, + &progress_thread_run, + &atomic_counter, + 1, + 1, + non_cached_files_to_check.len(), + CheckingMethod::None, + ); - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - let videos_to_check = non_cached_files_to_check.len(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 1, - max_stage: 1, - videos_checked: atomic_file_counter.load(Ordering::Relaxed), - videos_to_check, - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END let mut vec_file_entry: Vec = non_cached_files_to_check .par_iter() .map(|file_entry| { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); + atomic_counter.fetch_add(1, Ordering::Relaxed); if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { check_was_stopped.store(true, Ordering::Relaxed); return None; @@ -538,9 +433,7 @@ impl SimilarVideos { .while_some() .collect::>(); - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - reading data from files in parallel"); let hash_map_modification = SystemTime::now(); @@ -579,8 +472,32 @@ impl SimilarVideos { Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - saving data to files"); let hash_map_modification = SystemTime::now(); - let match_group = vid_dup_finder_lib::search(vector_of_hashes, NormalizedTolerance::new(self.tolerance as f64 / 100.0f64)); + self.match_groups_of_videos(vector_of_hashes, &hashmap_with_file_entries); + self.remove_from_reference_folders(); + if self.use_reference_folders { + for (_fe, vector) in &self.similar_referenced_vectors { + self.information.number_of_duplicates += vector.len(); + self.information.number_of_groups += 1; + } + } else { + for vector in &self.similar_vectors { + self.information.number_of_duplicates += vector.len() - 1; + self.information.number_of_groups += 1; + } + } + + Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - selecting data from BtreeMap"); + + // Clean unused data + self.videos_hashes = Default::default(); + self.videos_to_check = Default::default(); + + true + } + + fn match_groups_of_videos(&mut self, vector_of_hashes: Vec, hashmap_with_file_entries: &HashMap) { + let match_group = vid_dup_finder_lib::search(vector_of_hashes, NormalizedTolerance::new(self.tolerance as f64 / 100.0f64)); let mut collected_similar_videos: Vec> = Default::default(); for i in match_group { let mut temp_vector: Vec = Vec::new(); @@ -602,7 +519,9 @@ impl SimilarVideos { } self.similar_vectors = collected_similar_videos; + } + fn remove_from_reference_folders(&mut self) { if self.use_reference_folders { let mut similar_vector = Default::default(); mem::swap(&mut self.similar_vectors, &mut similar_vector); @@ -628,26 +547,6 @@ impl SimilarVideos { }) .collect::)>>(); } - - if self.use_reference_folders { - for (_fe, vector) in &self.similar_referenced_vectors { - self.information.number_of_duplicates += vector.len(); - self.information.number_of_groups += 1; - } - } else { - for vector in &self.similar_vectors { - self.information.number_of_duplicates += vector.len() - 1; - self.information.number_of_groups += 1; - } - } - - Common::print_time(hash_map_modification, SystemTime::now(), "sort_videos - selecting data from BtreeMap"); - - // Clean unused data - self.videos_hashes = Default::default(); - self.videos_to_check = Default::default(); - - true } /// Set included dir which needs to be relative, exists etc. diff --git a/czkawka_core/src/temporary.rs b/czkawka_core/src/temporary.rs index a94dfd3..fa07713 100644 --- a/czkawka_core/src/temporary.rs +++ b/czkawka_core/src/temporary.rs @@ -1,30 +1,38 @@ -use std::fs::{File, Metadata}; +use std::fs; +use std::fs::{DirEntry, File, Metadata}; use std::io::prelude::*; use std::io::BufWriter; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread::sleep; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use std::{fs, thread}; +use std::time::SystemTime; use crossbeam_channel::Receiver; +use futures::channel::mpsc::UnboundedSender; use rayon::prelude::*; -use crate::common::{Common, LOOP_DURATION}; +use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, Common}; +use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData}; use crate::common_directory::Directories; use crate::common_items::ExcludedItems; use crate::common_messages::Messages; use crate::common_traits::*; -use crate::flc; -use crate::localizer_core::generate_translation_hashmap; -#[derive(Debug)] -pub struct ProgressData { - pub current_stage: u8, - pub max_stage: u8, - pub files_checked: usize, -} +const TEMP_EXTENSIONS: &[&str] = &[ + "#", + "thumbs.db", + ".bak", + "~", + ".tmp", + ".temp", + ".ds_store", + ".crdownload", + ".part", + ".cache", + ".dmp", + ".download", + ".partial", +]; #[derive(Eq, PartialEq, Clone, Debug, Copy)] pub enum DeleteMethod { @@ -79,7 +87,7 @@ impl Temporary { } /// Finding temporary files, save results to internal struct variables - pub fn find_temporary_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) { + pub fn find_temporary_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) { self.directories.optimize_directories(self.recursive_search, &mut self.text_messages); if !self.check_files(stop_receiver, progress_sender) { self.stopped_search = true; @@ -134,7 +142,7 @@ impl Temporary { self.excluded_items.set_excluded_items(excluded_items, &mut self.text_messages); } - fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender>) -> bool { + fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender>) -> bool { let start_time: SystemTime = SystemTime::now(); let mut folders_to_check: Vec = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector @@ -143,38 +151,13 @@ impl Temporary { folders_to_check.push(id.clone()); } - //// PROGRESS THREAD START let progress_thread_run = Arc::new(AtomicBool::new(true)); - - let atomic_file_counter = Arc::new(AtomicUsize::new(0)); - - let progress_thread_handle = if let Some(progress_sender) = progress_sender { - let progress_send = progress_sender.clone(); - let progress_thread_run = progress_thread_run.clone(); - let atomic_file_counter = atomic_file_counter.clone(); - thread::spawn(move || loop { - progress_send - .unbounded_send(ProgressData { - current_stage: 0, - max_stage: 0, - files_checked: atomic_file_counter.load(Ordering::Relaxed), - }) - .unwrap(); - if !progress_thread_run.load(Ordering::Relaxed) { - break; - } - sleep(Duration::from_millis(LOOP_DURATION as u64)); - }) - } else { - thread::spawn(|| {}) - }; - //// PROGRESS THREAD END + let atomic_counter = Arc::new(AtomicUsize::new(0)); + let progress_thread_handle = prepare_thread_handler_common(progress_sender, &progress_thread_run, &atomic_counter, 0, 0, 0, CheckingMethod::None); while !folders_to_check.is_empty() { if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() { - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); return false; } @@ -184,129 +167,31 @@ impl Temporary { let mut dir_result = vec![]; let mut warnings = vec![]; let mut fe_result = vec![]; - // Read current dir children - let read_dir = match fs::read_dir(current_folder) { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_open_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - return (dir_result, warnings, fe_result); - } + + let Some(read_dir) = common_read_dir(current_folder, &mut warnings) else { + return (dir_result, warnings, fe_result); }; // Check every sub folder/file/link etc. - 'dir: for entry in read_dir { - let entry_data = match entry { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_entry_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } - }; - let metadata: Metadata = match entry_data.metadata() { - Ok(t) => t, - Err(e) => { - warnings.push(flc!( - "core_cannot_read_metadata_dir", - generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())]) - )); - continue 'dir; - } + for entry in read_dir { + let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else { + continue; }; + if metadata.is_dir() { - if !self.recursive_search { - continue 'dir; - } - - let next_folder = current_folder.join(entry_data.file_name()); - if self.directories.is_excluded(&next_folder) { - continue 'dir; - } - - if self.excluded_items.is_excluded(&next_folder) { - continue 'dir; - } - - #[cfg(target_family = "unix")] - if self.directories.exclude_other_filesystems() { - match self.directories.is_on_other_filesystems(&next_folder) { - Ok(true) => continue 'dir, - Err(e) => warnings.push(e.to_string()), - _ => (), - } - } - - dir_result.push(next_folder); + check_folder_children( + &mut dir_result, + &mut warnings, + current_folder, + entry_data, + self.recursive_search, + &self.directories, + &self.excluded_items, + ); } else if metadata.is_file() { - atomic_file_counter.fetch_add(1, Ordering::Relaxed); - - let file_name_lowercase: String = match entry_data.file_name().into_string() { - Ok(t) => t, - Err(_inspected) => { - warnings.push(flc!( - "core_file_not_utf8_name", - generate_translation_hashmap(vec![("name", entry_data.path().display().to_string())]) - )); - continue 'dir; - } + if let Some(file_entry) = self.get_file_entry(&metadata, &atomic_counter, entry_data, &mut warnings, current_folder) { + fe_result.push(file_entry); } - .to_lowercase(); - - if ![ - "#", - "thumbs.db", - ".bak", - "~", - ".tmp", - ".temp", - ".ds_store", - ".crdownload", - ".part", - ".cache", - ".dmp", - ".download", - ".partial", - ] - .iter() - .any(|f| file_name_lowercase.ends_with(f)) - { - continue 'dir; - } - let current_file_name = current_folder.join(entry_data.file_name()); - if self.excluded_items.is_excluded(¤t_file_name) { - continue 'dir; - } - - // Creating new file entry - let fe: FileEntry = FileEntry { - path: current_file_name.clone(), - modified_date: match metadata.modified() { - Ok(t) => match t.duration_since(UNIX_EPOCH) { - Ok(d) => d.as_secs(), - Err(_inspected) => { - warnings.push(flc!( - "core_file_modified_before_epoch", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string())]) - )); - 0 - } - }, - Err(e) => { - warnings.push(flc!( - "core_file_no_modification_date", - generate_translation_hashmap(vec![("name", current_file_name.display().to_string()), ("reason", e.to_string())]) - )); - 0 - } // Permissions Denied - }, - }; - - fe_result.push(fe); } } (dir_result, warnings, fe_result) @@ -326,14 +211,40 @@ impl Temporary { } } - // End thread which send info to gui - progress_thread_run.store(false, Ordering::Relaxed); - progress_thread_handle.join().unwrap(); + send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); self.information.number_of_temporary_files = self.temporary_files.len(); Common::print_time(start_time, SystemTime::now(), "check_files_size"); true } + pub fn get_file_entry( + &self, + metadata: &Metadata, + atomic_counter: &Arc, + entry_data: &DirEntry, + warnings: &mut Vec, + current_folder: &Path, + ) -> Option { + atomic_counter.fetch_add(1, Ordering::Relaxed); + + let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else { + return None; + }; + + if !TEMP_EXTENSIONS.iter().any(|f| file_name_lowercase.ends_with(f)) { + return None; + } + let current_file_name = current_folder.join(entry_data.file_name()); + if self.excluded_items.is_excluded(¤t_file_name) { + return None; + } + + // Creating new file entry + Some(FileEntry { + path: current_file_name.clone(), + modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false), + }) + } /// Function to delete files, from filed Vector fn delete_files(&mut self) { diff --git a/czkawka_gui/Cargo.toml b/czkawka_gui/Cargo.toml index ba16383..93c6580 100644 --- a/czkawka_gui/Cargo.toml +++ b/czkawka_gui/Cargo.toml @@ -11,49 +11,49 @@ repository = "https://github.com/qarmin/czkawka" [dependencies] gdk4 = "0.6.3" -glib = "0.17.5" +glib = "0.17.9" -humansize = "2.1.3" +humansize = "2.1" chrono = "0.4.24" # Used for sending stop signal across threads -crossbeam-channel = "0.5.7" +crossbeam-channel = "0.5.8" # To get information about progress futures = "0.3.28" # For saving/loading config files to specific directories -directories-next = "2.0.0" +directories-next = "2.0" # For opening files -open = "4.0.1" +open = "4.1" # To get image preview -image = "0.24.6" +image = "0.24" # To be able to use custom select -regex = "1.7.3" +regex = "1.8" # To get image_hasher types -image_hasher = "1.1.2" +image_hasher = "1.1" # Move files to trash -trash = "3.0.1" +trash = "3.0" # For moving files(why std::fs doesn't have such features) -fs_extra = "1.3.0" +fs_extra = "1.3" # Language -i18n-embed = { version = "0.13.8", features = ["fluent-system", "desktop-requester"] } -i18n-embed-fl = "0.6.6" -rust-embed = "6.6.1" -once_cell = "1.17.1" +i18n-embed = { version = "0.13", features = ["fluent-system", "desktop-requester"] } +i18n-embed-fl = "0.6" +rust-embed = "6.6" +once_cell = "1.17" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["combaseapi", "objbase", "shobjidl_core", "windef", "winerror", "wtypesbase", "winuser"] } [dependencies.gtk4] -version = "0.6.4" +version = "0.6" default-features = false features = ["v4_6"] diff --git a/czkawka_gui/src/compute_results.rs b/czkawka_gui/src/compute_results.rs index e1f3a8e..9ec02d3 100644 --- a/czkawka_gui/src/compute_results.rs +++ b/czkawka_gui/src/compute_results.rs @@ -1345,6 +1345,7 @@ fn vector_sort_unstable_entry_by_path(vector: &Vec) -> Vec vector.clone() } } + fn vector_sort_simple_unstable_entry_by_path(vector: &[FileEntry]) -> Vec { let mut vector = vector.to_owned(); vector.sort_unstable_by_key(|e| { diff --git a/czkawka_gui/src/connect_things/connect_button_move.rs b/czkawka_gui/src/connect_things/connect_button_move.rs index 734f8b5..2c1d500 100644 --- a/czkawka_gui/src/connect_things/connect_button_move.rs +++ b/czkawka_gui/src/connect_things/connect_button_move.rs @@ -1,5 +1,6 @@ use std::path::{Path, PathBuf}; +use fs_extra::dir::CopyOptions; use gtk4::prelude::*; use gtk4::{ResponseType, TreePath}; @@ -199,7 +200,7 @@ fn move_files_common( let thing = get_full_name_from_path_name(&path, &file_name); let destination_file = destination_folder.join(file_name); if Path::new(&thing).is_dir() { - if let Err(e) = fs_extra::dir::move_dir(&thing, &destination_file, &fs_extra::dir::CopyOptions::new()) { + if let Err(e) = fs_extra::dir::move_dir(&thing, &destination_file, &CopyOptions::new()) { messages += flg!("move_folder_failed", generate_translation_hashmap(vec![("name", thing), ("reason", e.to_string())])).as_str(); messages += "\n"; continue 'next_result; diff --git a/czkawka_gui/src/connect_things/connect_button_search.rs b/czkawka_gui/src/connect_things/connect_button_search.rs index 948753a..4f7bc62 100644 --- a/czkawka_gui/src/connect_things/connect_button_search.rs +++ b/czkawka_gui/src/connect_things/connect_button_search.rs @@ -2,13 +2,14 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::thread; +use futures::channel::mpsc::UnboundedSender; use glib::Sender; use gtk4::prelude::*; use czkawka_core::bad_extensions::BadExtensions; use czkawka_core::big_file::BigFile; use czkawka_core::broken_files::{BrokenFiles, CheckedTypes}; -use czkawka_core::common_dir_traversal; +use czkawka_core::common_dir_traversal::ProgressData; use czkawka_core::duplicate::DuplicateFinder; use czkawka_core::empty_files::EmptyFiles; use czkawka_core::empty_folder::EmptyFolder; @@ -17,7 +18,6 @@ use czkawka_core::same_music::{MusicSimilarity, SameMusic}; use czkawka_core::similar_images::SimilarImages; use czkawka_core::similar_videos::SimilarVideos; use czkawka_core::temporary::Temporary; -use czkawka_core::*; use crate::gui_structs::gui_data::GuiData; use crate::help_combo_box::{ @@ -33,17 +33,17 @@ use crate::{flg, DEFAULT_MAXIMAL_FILE_SIZE, DEFAULT_MINIMAL_CACHE_SIZE, DEFAULT_ pub fn connect_button_search( gui_data: &GuiData, glib_stop_sender: Sender, - futures_sender_duplicate_files: futures::channel::mpsc::UnboundedSender, - futures_sender_empty_files: futures::channel::mpsc::UnboundedSender, - futures_sender_empty_folder: futures::channel::mpsc::UnboundedSender, - futures_sender_big_file: futures::channel::mpsc::UnboundedSender, - futures_sender_same_music: futures::channel::mpsc::UnboundedSender, - futures_sender_similar_images: futures::channel::mpsc::UnboundedSender, - futures_sender_similar_videos: futures::channel::mpsc::UnboundedSender, - futures_sender_temporary: futures::channel::mpsc::UnboundedSender, - futures_sender_invalid_symlinks: futures::channel::mpsc::UnboundedSender, - futures_sender_broken_files: futures::channel::mpsc::UnboundedSender, - futures_sender_bad_extensions: futures::channel::mpsc::UnboundedSender, + futures_sender_duplicate_files: UnboundedSender, + futures_sender_empty_files: UnboundedSender, + futures_sender_empty_folder: UnboundedSender, + futures_sender_big_file: UnboundedSender, + futures_sender_same_music: UnboundedSender, + futures_sender_similar_images: UnboundedSender, + futures_sender_similar_videos: UnboundedSender, + futures_sender_temporary: UnboundedSender, + futures_sender_invalid_symlinks: UnboundedSender, + futures_sender_broken_files: UnboundedSender, + futures_sender_bad_extensions: UnboundedSender, ) { let check_button_settings_one_filesystem = gui_data.settings.check_button_settings_one_filesystem.clone(); let combo_box_image_hash_size = gui_data.main_notebook.combo_box_image_hash_size.clone(); diff --git a/czkawka_gui/src/connect_things/connect_popovers_sort.rs b/czkawka_gui/src/connect_things/connect_popovers_sort.rs index 62156cc..ca8c5ee 100644 --- a/czkawka_gui/src/connect_things/connect_popovers_sort.rs +++ b/czkawka_gui/src/connect_things/connect_popovers_sort.rs @@ -119,6 +119,7 @@ pub fn connect_popover_sort(gui_data: &GuiData) { #[cfg(test)] mod test { + use glib::types::Type; use gtk4::prelude::*; use gtk4::{Popover, TreeView}; @@ -126,7 +127,7 @@ mod test { #[gtk4::test] fn test_sort_iters() { - let columns_types: &[glib::types::Type] = &[glib::types::Type::U32, glib::types::Type::STRING]; + let columns_types: &[Type] = &[Type::U32, Type::STRING]; let list_store = gtk4::ListStore::new(columns_types); let values_to_add: &[&[(u32, &dyn ToValue)]] = &[&[(0, &2), (1, &"AAA")], &[(0, &3), (1, &"CCC")], &[(0, &1), (1, &"BBB")]]; @@ -156,7 +157,7 @@ mod test { #[gtk4::test] pub fn test_popover_sort_general_simple() { - let columns_types: &[glib::types::Type] = &[glib::types::Type::BOOL, glib::types::Type::STRING]; + let columns_types: &[Type] = &[Type::BOOL, Type::STRING]; let list_store = gtk4::ListStore::new(columns_types); let tree_view = TreeView::builder().model(&list_store).build(); let popover = Popover::new(); @@ -179,7 +180,7 @@ mod test { #[gtk4::test] pub fn test_popover_sort_general() { - let columns_types: &[glib::types::Type] = &[glib::types::Type::BOOL, glib::types::Type::STRING]; + let columns_types: &[Type] = &[Type::BOOL, Type::STRING]; let list_store = gtk4::ListStore::new(columns_types); let tree_view = TreeView::builder().model(&list_store).build(); let popover = Popover::new(); diff --git a/czkawka_gui/src/connect_things/connect_progress_window.rs b/czkawka_gui/src/connect_things/connect_progress_window.rs index 825281c..1a80f03 100644 --- a/czkawka_gui/src/connect_things/connect_progress_window.rs +++ b/czkawka_gui/src/connect_things/connect_progress_window.rs @@ -2,8 +2,8 @@ use futures::channel::mpsc::UnboundedReceiver; use futures::StreamExt; use gtk4::prelude::*; +use czkawka_core::common_dir_traversal; use czkawka_core::common_dir_traversal::ProgressData; -use czkawka_core::{big_file, broken_files, common_dir_traversal, similar_images, similar_videos, temporary}; use crate::flg; use crate::gui_structs::gui_data::GuiData; @@ -16,13 +16,13 @@ pub fn connect_progress_window( mut futures_receiver_duplicate_files: UnboundedReceiver, mut futures_receiver_empty_files: UnboundedReceiver, mut futures_receiver_empty_folder: UnboundedReceiver, - mut futures_receiver_big_files: UnboundedReceiver, + mut futures_receiver_big_files: UnboundedReceiver, mut futures_receiver_same_music: UnboundedReceiver, - mut futures_receiver_similar_images: UnboundedReceiver, - mut futures_receiver_similar_videos: UnboundedReceiver, - mut futures_receiver_temporary: UnboundedReceiver, + mut futures_receiver_similar_images: UnboundedReceiver, + mut futures_receiver_similar_videos: UnboundedReceiver, + mut futures_receiver_temporary: UnboundedReceiver, mut futures_receiver_invalid_symlinks: UnboundedReceiver, - mut futures_receiver_broken_files: UnboundedReceiver, + mut futures_receiver_broken_files: UnboundedReceiver, mut futures_receiver_bad_extensions: UnboundedReceiver, ) { let main_context = glib::MainContext::default(); @@ -175,7 +175,7 @@ pub fn connect_progress_window( while let Some(item) = futures_receiver_big_files.next().await { label_stage.set_text(&flg!( "progress_scanning_general_file", - generate_translation_hashmap(vec![("file_number", item.files_checked.to_string())]) + generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())]) )); taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE); } @@ -257,18 +257,18 @@ pub fn connect_progress_window( progress_bar_current_stage.hide(); label_stage.set_text(&flg!( "progress_scanning_general_file", - generate_translation_hashmap(vec![("file_number", item.images_checked.to_string())]) + generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())]) )); taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE); } 1 => { progress_bar_current_stage.show(); - if item.images_to_check != 0 { - progress_bar_all_stages.set_fraction((1f64 + (item.images_checked) as f64 / item.images_to_check as f64) / (item.max_stage + 1) as f64); - progress_bar_current_stage.set_fraction((item.images_checked) as f64 / item.images_to_check as f64); + if item.entries_to_check != 0 { + progress_bar_all_stages.set_fraction((1f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64); + progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64); taskbar_state.borrow().set_progress_value( - (item.images_to_check + item.images_checked) as u64, - item.images_to_check as u64 * (item.max_stage + 1) as u64, + (item.entries_to_check + item.entries_checked) as u64, + item.entries_to_check as u64 * (item.max_stage + 1) as u64, ); } else { progress_bar_all_stages.set_fraction((item.current_stage as f64) / (item.max_stage + 1) as f64); @@ -277,17 +277,17 @@ pub fn connect_progress_window( } label_stage.set_text(&flg!( "progress_scanning_image", - generate_translation_hashmap(vec![("file_checked", item.images_checked.to_string()), ("all_files", item.images_to_check.to_string())]) + generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())]) )); } 2 => { progress_bar_current_stage.show(); - if item.images_to_check != 0 { - progress_bar_all_stages.set_fraction((2f64 + (item.images_checked) as f64 / item.images_to_check as f64) / (item.max_stage + 1) as f64); - progress_bar_current_stage.set_fraction((item.images_checked) as f64 / item.images_to_check as f64); + if item.entries_to_check != 0 { + progress_bar_all_stages.set_fraction((2f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64); + progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64); taskbar_state.borrow().set_progress_value( - (item.images_to_check + item.images_checked) as u64, - item.images_to_check as u64 * (item.max_stage + 1) as u64, + (item.entries_to_check + item.entries_checked) as u64, + item.entries_to_check as u64 * (item.max_stage + 1) as u64, ); } else { progress_bar_all_stages.set_fraction((item.current_stage as f64) / (item.max_stage + 1) as f64); @@ -296,7 +296,7 @@ pub fn connect_progress_window( } label_stage.set_text(&flg!( "progress_comparing_image_hashes", - generate_translation_hashmap(vec![("file_checked", item.images_checked.to_string()), ("all_files", item.images_to_check.to_string())]) + generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())]) )); } _ => { @@ -320,18 +320,18 @@ pub fn connect_progress_window( progress_bar_current_stage.hide(); label_stage.set_text(&flg!( "progress_scanning_general_file", - generate_translation_hashmap(vec![("file_number", item.videos_checked.to_string())]) + generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())]) )); taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE); } 1 => { progress_bar_current_stage.show(); - if item.videos_to_check != 0 { - progress_bar_all_stages.set_fraction((1f64 + (item.videos_checked) as f64 / item.videos_to_check as f64) / (item.max_stage + 1) as f64); - progress_bar_current_stage.set_fraction((item.videos_checked) as f64 / item.videos_to_check as f64); + if item.entries_to_check != 0 { + progress_bar_all_stages.set_fraction((1f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64); + progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64); taskbar_state.borrow().set_progress_value( - (item.videos_to_check + item.videos_checked) as u64, - item.videos_to_check as u64 * (item.max_stage + 1) as u64, + (item.entries_to_check + item.entries_checked) as u64, + item.entries_to_check as u64 * (item.max_stage + 1) as u64, ); } else { progress_bar_all_stages.set_fraction((1f64) / (item.max_stage + 1) as f64); @@ -340,7 +340,7 @@ pub fn connect_progress_window( } label_stage.set_text(&flg!( "progress_scanning_video", - generate_translation_hashmap(vec![("file_checked", item.videos_checked.to_string()), ("all_files", item.videos_to_check.to_string())]) + generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())]) )); } _ => { @@ -359,7 +359,7 @@ pub fn connect_progress_window( while let Some(item) = futures_receiver_temporary.next().await { label_stage.set_text(&flg!( "progress_scanning_general_file", - generate_translation_hashmap(vec![("file_number", item.files_checked.to_string())]) + generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())]) )); taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE); } @@ -394,18 +394,19 @@ pub fn connect_progress_window( progress_bar_current_stage.hide(); label_stage.set_text(&flg!( "progress_scanning_general_file", - generate_translation_hashmap(vec![("file_number", item.files_checked.to_string())]) + generate_translation_hashmap(vec![("file_number", item.entries_checked.to_string())]) )); taskbar_state.borrow().set_progress_state(TBPF_INDETERMINATE); } 1 => { progress_bar_current_stage.show(); - if item.files_to_check != 0 { - progress_bar_all_stages.set_fraction((1f64 + (item.files_checked) as f64 / item.files_to_check as f64) / (item.max_stage + 1) as f64); - progress_bar_current_stage.set_fraction((item.files_checked) as f64 / item.files_to_check as f64); - taskbar_state - .borrow() - .set_progress_value((item.files_to_check + item.files_checked) as u64, item.files_to_check as u64 * (item.max_stage + 1) as u64); + if item.entries_to_check != 0 { + progress_bar_all_stages.set_fraction((1f64 + (item.entries_checked) as f64 / item.entries_to_check as f64) / (item.max_stage + 1) as f64); + progress_bar_current_stage.set_fraction((item.entries_checked) as f64 / item.entries_to_check as f64); + taskbar_state.borrow().set_progress_value( + (item.entries_to_check + item.entries_checked) as u64, + item.entries_to_check as u64 * (item.max_stage + 1) as u64, + ); } else { progress_bar_all_stages.set_fraction((1f64) / (item.max_stage + 1) as f64); progress_bar_current_stage.set_fraction(0f64); @@ -413,7 +414,7 @@ pub fn connect_progress_window( } label_stage.set_text(&flg!( "progress_scanning_broken_files", - generate_translation_hashmap(vec![("file_checked", item.files_checked.to_string()), ("all_files", item.files_to_check.to_string())]) + generate_translation_hashmap(vec![("file_checked", item.entries_checked.to_string()), ("all_files", item.entries_to_check.to_string())]) )); } _ => { diff --git a/czkawka_gui/src/gui_structs/gui_data.rs b/czkawka_gui/src/gui_structs/gui_data.rs index ef96646..f00e2d3 100644 --- a/czkawka_gui/src/gui_structs/gui_data.rs +++ b/czkawka_gui/src/gui_structs/gui_data.rs @@ -1,5 +1,6 @@ use std::cell::RefCell; use std::collections::HashMap; +use std::io::BufReader; use std::rc::Rc; use crossbeam_channel::bounded; @@ -122,7 +123,7 @@ impl GuiData { window_main.set_title(Some(&flg!("window_main_title"))); window_main.show(); - let pixbuf = Pixbuf::from_read(std::io::BufReader::new(ICON_ABOUT)).unwrap(); + let pixbuf = Pixbuf::from_read(BufReader::new(ICON_ABOUT)).unwrap(); window_main.set_application(Some(application)); diff --git a/czkawka_gui/src/help_functions.rs b/czkawka_gui/src/help_functions.rs index f530101..e3af425 100644 --- a/czkawka_gui/src/help_functions.rs +++ b/czkawka_gui/src/help_functions.rs @@ -1,5 +1,6 @@ use std::cmp::Ordering; use std::collections::HashMap; +use std::io::BufReader; use std::path::PathBuf; use gdk4::gdk_pixbuf::{InterpType, Pixbuf}; @@ -714,7 +715,7 @@ const TYPE_OF_INTERPOLATION: InterpType = InterpType::Tiles; pub fn set_icon_of_button>(button: &P, data: &'static [u8]) { let image = get_custom_image_from_widget(&button.clone()); - let pixbuf = Pixbuf::from_read(std::io::BufReader::new(data)).unwrap(); + let pixbuf = Pixbuf::from_read(BufReader::new(data)).unwrap(); let pixbuf = pixbuf.scale_simple(SIZE_OF_ICON, SIZE_OF_ICON, TYPE_OF_INTERPOLATION).unwrap(); image.set_from_pixbuf(Some(&pixbuf)); } @@ -778,6 +779,7 @@ pub fn scale_step_function(scale: >k4::Scale, _scroll_type: ScrollType, value: #[cfg(test)] mod test { + use glib::types::Type; use gtk4::prelude::*; use gtk4::Orientation; use image::DynamicImage; @@ -789,7 +791,7 @@ mod test { #[gtk4::test] fn test_check_if_list_store_column_have_all_same_values() { - let columns_types: &[glib::types::Type] = &[glib::types::Type::BOOL]; + let columns_types: &[Type] = &[Type::BOOL]; let list_store = gtk4::ListStore::new(columns_types); list_store.clear(); @@ -823,7 +825,7 @@ mod test { #[gtk4::test] fn test_check_if_value_is_in_list_store() { - let columns_types: &[glib::types::Type] = &[glib::types::Type::STRING]; + let columns_types: &[Type] = &[Type::STRING]; let list_store = gtk4::ListStore::new(columns_types); let values_to_add: &[(u32, &dyn ToValue)] = &[(0, &"Koczkodan"), (0, &"Kachir")]; for i in values_to_add { @@ -833,7 +835,7 @@ mod test { assert!(check_if_value_is_in_list_store(&list_store, 0, "Kachir")); assert!(!check_if_value_is_in_list_store(&list_store, 0, "Koczkodan2")); - let columns_types: &[glib::types::Type] = &[glib::types::Type::STRING, glib::types::Type::STRING]; + let columns_types: &[Type] = &[Type::STRING, Type::STRING]; let list_store = gtk4::ListStore::new(columns_types); let values_to_add: &[&[(u32, &dyn ToValue)]] = &[&[(0, &"Koczkodan"), (1, &"Krakus")], &[(0, &"Kachir"), (1, &"Wodnica")]]; for i in values_to_add { diff --git a/czkawka_gui/src/initialize_gui.rs b/czkawka_gui/src/initialize_gui.rs index 2c4690b..527d180 100644 --- a/czkawka_gui/src/initialize_gui.rs +++ b/czkawka_gui/src/initialize_gui.rs @@ -3,6 +3,7 @@ use std::path::Path; use std::rc::Rc; use gdk4::gdk_pixbuf::Pixbuf; +use glib::types::Type; use gtk4::gdk_pixbuf::InterpType; use gtk4::prelude::*; use gtk4::{CheckButton, Image, SelectionMode, TextView, TreeView}; @@ -304,9 +305,9 @@ pub fn initialize_gui(gui_data: &mut GuiData) { let evk = gui_data.upper_notebook.evk_tree_view_included_directories.clone(); let gc = gui_data.upper_notebook.gc_tree_view_included_directories.clone(); - let col_types: [glib::types::Type; 2] = [ - glib::types::Type::STRING, // Path - glib::types::Type::BOOL, // ReferenceButton + let col_types: [Type; 2] = [ + Type::STRING, // Path + Type::BOOL, // ReferenceButton ]; let list_store: gtk4::ListStore = gtk4::ListStore::new(&col_types); @@ -341,7 +342,7 @@ pub fn initialize_gui(gui_data: &mut GuiData) { let evk = gui_data.upper_notebook.evk_tree_view_excluded_directories.clone(); let gc = gui_data.upper_notebook.gc_tree_view_excluded_directories.clone(); - let col_types: [glib::types::Type; 1] = [glib::types::Type::STRING]; + let col_types: [Type; 1] = [Type::STRING]; let list_store: gtk4::ListStore = gtk4::ListStore::new(&col_types); tree_view.set_model(Some(&list_store)); diff --git a/czkawka_gui/src/main.rs b/czkawka_gui/src/main.rs index 604c914..c45ef87 100644 --- a/czkawka_gui/src/main.rs +++ b/czkawka_gui/src/main.rs @@ -8,6 +8,8 @@ use std::env; use std::ffi::OsString; +use futures::channel::mpsc; +use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender}; use gtk4::gio::ApplicationFlags; use gtk4::prelude::*; use gtk4::Application; @@ -32,6 +34,7 @@ use connect_things::connect_settings::*; use connect_things::connect_show_hide_ui::*; use connect_things::connect_similar_image_size_change::*; use czkawka_core::common::{get_number_of_threads, set_number_of_threads}; +use czkawka_core::common_dir_traversal::ProgressData; use czkawka_core::*; use gui_structs::gui_data::*; @@ -80,50 +83,17 @@ fn build_ui(application: &Application, arguments: &[OsString]) { let (glib_stop_sender, glib_stop_receiver) = glib::MainContext::channel(glib::PRIORITY_DEFAULT); // Futures progress report - let (futures_sender_duplicate_files, futures_receiver_duplicate_files): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_empty_files, futures_receiver_empty_files): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_empty_folder, futures_receiver_empty_folder): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_big_file, futures_receiver_big_files): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_same_music, futures_receiver_same_music): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_similar_images, futures_receiver_similar_images): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_similar_videos, futures_receiver_similar_videos): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_temporary, futures_receiver_temporary): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_invalid_symlinks, futures_receiver_invalid_symlinks): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_broken_files, futures_receiver_broken_files): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); - let (futures_sender_bad_extensions, futures_receiver_bad_extensions): ( - futures::channel::mpsc::UnboundedSender, - futures::channel::mpsc::UnboundedReceiver, - ) = futures::channel::mpsc::unbounded(); + let (futures_sender_duplicate_files, futures_receiver_duplicate_files): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_empty_files, futures_receiver_empty_files): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_empty_folder, futures_receiver_empty_folder): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_big_file, futures_receiver_big_files): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_same_music, futures_receiver_same_music): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_similar_images, futures_receiver_similar_images): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_similar_videos, futures_receiver_similar_videos): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_temporary, futures_receiver_temporary): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_invalid_symlinks, futures_receiver_invalid_symlinks): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_broken_files, futures_receiver_broken_files): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); + let (futures_sender_bad_extensions, futures_receiver_bad_extensions): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded(); initialize_gui(&mut gui_data); validate_notebook_data(&gui_data); // Must be run after initialization of gui, to check if everything was properly setup diff --git a/czkawka_gui/src/notebook_info.rs b/czkawka_gui/src/notebook_info.rs index 1438be0..cd7c8e1 100644 --- a/czkawka_gui/src/notebook_info.rs +++ b/czkawka_gui/src/notebook_info.rs @@ -1,3 +1,5 @@ +use glib::types::Type; + use crate::help_functions::{ BottomButtonsEnum, ColumnsBadExtensions, ColumnsBigFiles, ColumnsBrokenFiles, ColumnsDuplicates, ColumnsEmptyFiles, ColumnsEmptyFolders, ColumnsInvalidSymlinks, ColumnsSameMusic, ColumnsSimilarImages, ColumnsSimilarVideos, ColumnsTemporaryFiles, PopoverTypes, @@ -16,7 +18,7 @@ pub struct NotebookObject { pub column_size: Option, pub column_size_as_bytes: Option, pub column_modification_as_secs: Option, - pub columns_types: &'static [glib::types::Type], + pub columns_types: &'static [Type], pub bottom_buttons: &'static [BottomButtonsEnum], } @@ -41,17 +43,17 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: Some(ColumnsDuplicates::SizeAsBytes as i32), column_modification_as_secs: Some(ColumnsDuplicates::ModificationAsSecs as i32), columns_types: &[ - glib::types::Type::BOOL, // ActivatableSelectButton - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Size - glib::types::Type::U64, // SizeAsBytes - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs - glib::types::Type::STRING, // Color - glib::types::Type::BOOL, // IsHeader - glib::types::Type::STRING, // TextColor + Type::BOOL, // ActivatableSelectButton + Type::BOOL, // SelectionButton + Type::STRING, // Size + Type::U64, // SizeAsBytes + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // Modification + Type::U64, // ModificationAsSecs + Type::STRING, // Color + Type::BOOL, // IsHeader + Type::STRING, // TextColor ], bottom_buttons: &[ BottomButtonsEnum::Save, @@ -76,11 +78,11 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: None, column_modification_as_secs: None, columns_types: &[ - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs + Type::BOOL, // SelectionButton + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // Modification + Type::U64, // ModificationAsSecs ], bottom_buttons: &[BottomButtonsEnum::Save, BottomButtonsEnum::Delete, BottomButtonsEnum::Select, BottomButtonsEnum::Move], }, @@ -97,13 +99,13 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: None, column_modification_as_secs: None, columns_types: &[ - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Size - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // SizeAsBytes - glib::types::Type::U64, // ModificationAsSecs + Type::BOOL, // SelectionButton + Type::STRING, // Size + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // Modification + Type::U64, // SizeAsBytes + Type::U64, // ModificationAsSecs ], bottom_buttons: &[BottomButtonsEnum::Save, BottomButtonsEnum::Delete, BottomButtonsEnum::Select, BottomButtonsEnum::Move], }, @@ -120,11 +122,11 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: None, column_modification_as_secs: None, columns_types: &[ - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs + Type::BOOL, // SelectionButton + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // Modification + Type::U64, // ModificationAsSecs ], bottom_buttons: &[BottomButtonsEnum::Save, BottomButtonsEnum::Delete, BottomButtonsEnum::Select, BottomButtonsEnum::Move], }, @@ -141,11 +143,11 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: None, column_modification_as_secs: None, columns_types: &[ - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs + Type::BOOL, // SelectionButton + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // Modification + Type::U64, // ModificationAsSecs ], bottom_buttons: &[BottomButtonsEnum::Save, BottomButtonsEnum::Delete, BottomButtonsEnum::Select, BottomButtonsEnum::Move], }, @@ -162,19 +164,19 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: Some(ColumnsSimilarImages::SizeAsBytes as i32), column_modification_as_secs: Some(ColumnsSimilarImages::ModificationAsSecs as i32), columns_types: &[ - glib::types::Type::BOOL, // ActivatableSelectButton - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Similarity - glib::types::Type::STRING, // Size - glib::types::Type::U64, // SizeAsBytes - glib::types::Type::STRING, // Dimensions - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs - glib::types::Type::STRING, // Color - glib::types::Type::BOOL, // IsHeader - glib::types::Type::STRING, // TextColor + Type::BOOL, // ActivatableSelectButton + Type::BOOL, // SelectionButton + Type::STRING, // Similarity + Type::STRING, // Size + Type::U64, // SizeAsBytes + Type::STRING, // Dimensions + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // Modification + Type::U64, // ModificationAsSecs + Type::STRING, // Color + Type::BOOL, // IsHeader + Type::STRING, // TextColor ], bottom_buttons: &[ BottomButtonsEnum::Save, @@ -200,17 +202,17 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: Some(ColumnsSimilarVideos::SizeAsBytes as i32), column_modification_as_secs: Some(ColumnsSimilarVideos::ModificationAsSecs as i32), columns_types: &[ - glib::types::Type::BOOL, // ActivatableSelectButton - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Size - glib::types::Type::U64, // SizeAsBytes - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs - glib::types::Type::STRING, // Color - glib::types::Type::BOOL, // IsHeader - glib::types::Type::STRING, // TextColor + Type::BOOL, // ActivatableSelectButton + Type::BOOL, // SelectionButton + Type::STRING, // Size + Type::U64, // SizeAsBytes + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // Modification + Type::U64, // ModificationAsSecs + Type::STRING, // Color + Type::BOOL, // IsHeader + Type::STRING, // TextColor ], bottom_buttons: &[ BottomButtonsEnum::Save, @@ -235,24 +237,24 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: Some(ColumnsSameMusic::SizeAsBytes as i32), column_modification_as_secs: Some(ColumnsSameMusic::ModificationAsSecs as i32), columns_types: &[ - glib::types::Type::BOOL, // ActivatableSelectButton - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Size - glib::types::Type::U64, // SizeAsBytes - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // Title - glib::types::Type::STRING, // Artist - glib::types::Type::STRING, // Year - glib::types::Type::STRING, // Bitrate - glib::types::Type::U64, // BitrateAsNumber - glib::types::Type::STRING, // Length - glib::types::Type::STRING, // Genre - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs - glib::types::Type::STRING, // Color - glib::types::Type::BOOL, // IsHeader - glib::types::Type::STRING, // TextColor + Type::BOOL, // ActivatableSelectButton + Type::BOOL, // SelectionButton + Type::STRING, // Size + Type::U64, // SizeAsBytes + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // Title + Type::STRING, // Artist + Type::STRING, // Year + Type::STRING, // Bitrate + Type::U64, // BitrateAsNumber + Type::STRING, // Length + Type::STRING, // Genre + Type::STRING, // Modification + Type::U64, // ModificationAsSecs + Type::STRING, // Color + Type::BOOL, // IsHeader + Type::STRING, // TextColor ], bottom_buttons: &[ BottomButtonsEnum::Save, @@ -277,13 +279,13 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: None, column_modification_as_secs: None, columns_types: &[ - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // DestinationPath - glib::types::Type::STRING, // TypeOfError - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs + Type::BOOL, // SelectionButton + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // DestinationPath + Type::STRING, // TypeOfError + Type::STRING, // Modification + Type::U64, // ModificationAsSecs ], bottom_buttons: &[BottomButtonsEnum::Save, BottomButtonsEnum::Delete, BottomButtonsEnum::Select, BottomButtonsEnum::Move], }, @@ -300,12 +302,12 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: None, column_modification_as_secs: None, columns_types: &[ - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // ErrorType - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs + Type::BOOL, // SelectionButton + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // ErrorType + Type::STRING, // Modification + Type::U64, // ModificationAsSecs ], bottom_buttons: &[BottomButtonsEnum::Save, BottomButtonsEnum::Delete, BottomButtonsEnum::Select, BottomButtonsEnum::Move], }, @@ -322,13 +324,13 @@ pub static NOTEBOOKS_INFO: [NotebookObject; NUMBER_OF_NOTEBOOK_MAIN_TABS] = [ column_size_as_bytes: None, column_modification_as_secs: None, columns_types: &[ - glib::types::Type::BOOL, // SelectionButton - glib::types::Type::STRING, // Name - glib::types::Type::STRING, // Path - glib::types::Type::STRING, // CurrentExtension - glib::types::Type::STRING, // ProperExtensions - glib::types::Type::STRING, // Modification - glib::types::Type::U64, // ModificationAsSecs + Type::BOOL, // SelectionButton + Type::STRING, // Name + Type::STRING, // Path + Type::STRING, // CurrentExtension + Type::STRING, // ProperExtensions + Type::STRING, // Modification + Type::U64, // ModificationAsSecs ], bottom_buttons: &[BottomButtonsEnum::Save, BottomButtonsEnum::Delete, BottomButtonsEnum::Select, BottomButtonsEnum::Move], }, diff --git a/czkawka_gui/ui/czkawka.cmb b/czkawka_gui/ui/czkawka.cmb index c8bebf5..a162491 100755 --- a/czkawka_gui/ui/czkawka.cmb +++ b/czkawka_gui/ui/czkawka.cmb @@ -160,7 +160,7 @@ (5,122,"GtkCheckButton","check_button_music_genre",117,None,None,None,4), (5,123,"GtkCheckButton","check_button_music_length",117,None,None,None,5), (5,124,"GtkBox",None,116,None,None,None,1), - (5,125,"GtkCheckButton","check_button_music_approximate_comparison",124,None,None,None,None), + (5,125,"GtkCheckButton","check_button_music_approximate_comparison",124,None,None,None,2), (5,126,"GtkScrolledWindow","scrolled_window_same_music_finder",116,None,None,None,2), (5,127,"GtkLabel",None,115,None,None,None,None), (5,128,"GtkNotebookPage",None,56,None,None,None,8), @@ -235,6 +235,8 @@ (5,230,"GtkBox",None,229,None,None,None,None), (5,231,"GtkImage",None,230,None,None,None,None), (5,232,"GtkLabel","label_buttons_sort",230,None,None,None,1), + (5,234,"GtkLabel","label_audio_check_type",124,None,None,None,None), + (5,235,"GtkComboBoxText","combo_box_audio_check_type",124,None,None,None,1), (6,1,"GtkPopover","popover_right_click",None,None,None,None,None), (6,2,"GtkBox",None,1,None,None,None,None), (6,3,"GtkButton","buttons_popover_right_click_open_file",2,None,None,None,None), @@ -765,6 +767,8 @@ (5,230,"GtkWidget","halign","center",None,None,None,None,None), (5,231,"GtkImage","icon-name","image-missing",None,None,None,None,None), (5,232,"GtkLabel","label","SortMenu",None,None,None,None,None), + (5,234,"GtkLabel","label","Audio check type",None,None,None,None,None), + (5,234,"GtkWidget","margin-end","2",None,None,None,None,None), (6,1,"GtkPopover","child",None,None,None,None,None,2), (6,1,"GtkPopover","position","left",None,None,None,None,None), (6,2,"GtkOrientable","orientation","vertical",None,None,None,None,None), diff --git a/czkawka_gui/ui/main_window.ui b/czkawka_gui/ui/main_window.ui index 298f88a..3f9aec6 100644 --- a/czkawka_gui/ui/main_window.ui +++ b/czkawka_gui/ui/main_window.ui @@ -728,6 +728,15 @@ 2 5 5 + + + Audio check type + 2 + + + + + 1