CodeMirror/czkawka
1
0
Fork 0
mirror of synced 2024-05-13 17:02:31 +12:00
Code Issues Projects Releases Wiki Activity

Randoms (#1159)

Browse source
This commit is contained in:
Rafał Mikrut 2023-12-16 23:21:09 +01:00 committed by GitHub
parent 219f9f058b
commit bf78fc8b57
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 794 additions and 963 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/linux_cli_eyra.yml vendored
View file

@ -23,7 +23,7 @@ jobs:
# New versions of nightly rust may call new unimplemented in eyra functions, so use const version # New versions of nightly rust may call new unimplemented in eyra functions, so use const version
- name: Setup rust version - name: Setup rust version
run: rustup default nightly-2023-11-29 run: rustup default nightly-2023-12-14
- name: Add eyra - name: Add eyra
run: | run: |

178
Cargo.lock generated
View file

@ -278,13 +278,13 @@ dependencies = [
[[package]] [[package]]
name = "async-global-executor" name = "async-global-executor"
version = "2.4.0" version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4353121d5644cdf2beb5726ab752e79a8db1ebb52031770ec47db31d245526" checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c"
dependencies = [ dependencies = [
"async-channel 2.1.1", "async-channel 2.1.1",
"async-executor", "async-executor",
"async-io 2.2.1", "async-io 2.2.2",
"async-lock 3.2.0", "async-lock 3.2.0",
"blocking", "blocking",
"futures-lite 2.1.0", "futures-lite 2.1.0",
@ -313,9 +313,9 @@ dependencies = [
[[package]] [[package]]
name = "async-io" name = "async-io"
version = "2.2.1" version = "2.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6d3b15875ba253d1110c740755e246537483f152fa334f91abd7fe84c88b3ff" checksum = "6afaa937395a620e33dc6a742c593c01aced20aa376ffb0f628121198578ccc7"
dependencies = [ dependencies = [
"async-lock 3.2.0", "async-lock 3.2.0",
"cfg-if", "cfg-if",
@ -375,7 +375,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -384,7 +384,7 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5"
dependencies = [ dependencies = [
"async-io 2.2.1", "async-io 2.2.2",
"async-lock 2.8.0", "async-lock 2.8.0",
"atomic-waker", "atomic-waker",
"cfg-if", "cfg-if",
@ -436,7 +436,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -463,7 +463,7 @@ dependencies = [
"derive_utils", "derive_utils",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -512,7 +512,7 @@ dependencies = [
"regex", "regex",
"rustc-hash", "rustc-hash",
"shlex", "shlex",
"syn 2.0.39", "syn 2.0.41",
"which", "which",
] ]
@ -645,7 +645,7 @@ checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -874,7 +874,7 @@ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -994,7 +994,7 @@ checksum = "57aaaad9185d3bcb3afe63549d8ba60b2fb0ea8dc2da83f62dd56805edf56fd1"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -1128,9 +1128,9 @@ checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216"
[[package]] [[package]]
name = "crossbeam-channel" name = "crossbeam-channel"
version = "0.5.8" version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"crossbeam-utils", "crossbeam-utils",
@ -1138,9 +1138,9 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-deque" name = "crossbeam-deque"
version = "0.8.3" version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"crossbeam-epoch", "crossbeam-epoch",
@ -1149,22 +1149,21 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-epoch" name = "crossbeam-epoch"
version = "0.9.15" version = "0.9.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"cfg-if", "cfg-if",
"crossbeam-utils", "crossbeam-utils",
"memoffset 0.9.0", "memoffset 0.9.0",
"scopeguard",
] ]
[[package]] [[package]]
name = "crossbeam-queue" name = "crossbeam-queue"
version = "0.3.8" version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"crossbeam-utils", "crossbeam-utils",
@ -1172,9 +1171,9 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.16" version = "0.8.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
] ]
@ -1206,12 +1205,12 @@ dependencies = [
[[package]] [[package]]
name = "ctor" name = "ctor"
version = "0.2.5" version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583" checksum = "30d2b3721e861707777e3195b0158f950ae6dc4a27e4d02ff9f67e3eb3de199e"
dependencies = [ dependencies = [
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -1302,6 +1301,7 @@ dependencies = [
"log", "log",
"once_cell", "once_cell",
"open", "open",
"rayon",
"regex", "regex",
"rust-embed", "rust-embed",
"trash", "trash",
@ -1438,7 +1438,7 @@ checksum = "9abcad25e9720609ccb3dcdb795d845e37d8ce34183330a9f48b03a1a71c8e21"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -1487,7 +1487,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -1625,7 +1625,7 @@ checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -1636,7 +1636,7 @@ checksum = "c2ad8cef1d801a4686bfd8919f0b30eac4c8e48968c437a6405ded4fb5272d2b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -1964,7 +1964,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -2085,7 +2085,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -2339,7 +2339,7 @@ dependencies = [
"proc-macro-error", "proc-macro-error",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -2659,11 +2659,11 @@ dependencies = [
[[package]] [[package]]
name = "home" name = "home"
version = "0.5.5" version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
dependencies = [ dependencies = [
"windows-sys 0.48.0", "windows-sys 0.52.0",
] ]
[[package]] [[package]]
@ -2840,7 +2840,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9148dba9642f055cdda5060f2e82be4a4c1c4a046ea1d08970e9279220b7ed13" checksum = "9148dba9642f055cdda5060f2e82be4a4c1c4a046ea1d08970e9279220b7ed13"
dependencies = [ dependencies = [
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -2966,7 +2966,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"strsim", "strsim",
"syn 2.0.39", "syn 2.0.41",
"unic-langid", "unic-langid",
] ]
@ -2980,7 +2980,7 @@ dependencies = [
"i18n-config", "i18n-config",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -3386,9 +3386,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.150" version = "0.2.151"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
[[package]] [[package]]
name = "libflate" name = "libflate"
@ -3605,7 +3605,7 @@ checksum = "764b60e1ddd07e5665a6a17636a95cd7d8f3b86c73503a69c32979d05f72f3cf"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -3719,9 +3719,9 @@ dependencies = [
[[package]] [[package]]
name = "memmap2" name = "memmap2"
version = "0.9.0" version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "deaba38d7abf1d4cca21cc89e932e542ba2b9258664d2a9ef0e61512039c9375" checksum = "8f850157af41022bbb1b04ed15c011ce4d59520be82a4e3718b10c34b02cb85e"
dependencies = [ dependencies = [
"libc", "libc",
] ]
@ -3950,7 +3950,7 @@ dependencies = [
"proc-macro-crate 2.0.0", "proc-macro-crate 2.0.0",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -4260,7 +4260,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -4375,7 +4375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -4753,7 +4753,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"rust-embed-utils", "rust-embed-utils",
"syn 2.0.39", "syn 2.0.41",
"walkdir", "walkdir",
] ]
@ -4977,7 +4977,7 @@ checksum = "1729a30a469de249c6effc17ec8d039b0aa29b3af79b819b7f51cb6ab8046a90"
dependencies = [ dependencies = [
"ab_glyph", "ab_glyph",
"log", "log",
"memmap2 0.9.0", "memmap2 0.9.1",
"smithay-client-toolkit", "smithay-client-toolkit",
"tiny-skia", "tiny-skia",
] ]
@ -5026,7 +5026,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -5048,7 +5048,7 @@ checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -5247,7 +5247,7 @@ dependencies = [
"cursor-icon", "cursor-icon",
"libc", "libc",
"log", "log",
"memmap2 0.9.0", "memmap2 0.9.1",
"rustix 0.38.28", "rustix 0.38.28",
"thiserror", "thiserror",
"wayland-backend", "wayland-backend",
@ -5328,7 +5328,7 @@ dependencies = [
"foreign-types", "foreign-types",
"js-sys", "js-sys",
"log", "log",
"memmap2 0.9.0", "memmap2 0.9.1",
"objc", "objc",
"raw-window-handle 0.5.2", "raw-window-handle 0.5.2",
"redox_syscall 0.4.1", "redox_syscall 0.4.1",
@ -5433,7 +5433,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"rustversion", "rustversion",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -5648,9 +5648,9 @@ dependencies = [
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.39" version = "2.0.41"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -5717,22 +5717,22 @@ checksum = "f18aa187839b2bdb1ad2fa35ead8c4c2976b64e4363c386d45ac0f7ee85c9233"
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.50" version = "1.0.51"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7"
dependencies = [ dependencies = [
"thiserror-impl", "thiserror-impl",
] ]
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "1.0.50" version = "1.0.51"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -5933,7 +5933,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -6021,9 +6021,9 @@ checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4"
[[package]] [[package]]
name = "tuple" name = "tuple"
version = "0.5.1" version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39a40ba241047e1174c927dc5f61c141a166b938d61a2ff61838441368cc7d0e" checksum = "9bb9f6bd73479481158ba8ee3edf17aca93354623d13f02e96a2014fdbc1c37e"
dependencies = [ dependencies = [
"num-traits", "num-traits",
"serde", "serde",
@ -6066,28 +6066,29 @@ dependencies = [
[[package]] [[package]]
name = "uds_windows" name = "uds_windows"
version = "1.0.2" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce65604324d3cce9b966701489fbd0cf318cb1f7bd9dd07ac9a4ee6fb791930d" checksum = "89daebc3e6fd160ac4aa9fc8b3bf71e1f74fbf92367ae71fb83a037e8bf164b9"
dependencies = [ dependencies = [
"memoffset 0.9.0",
"tempfile", "tempfile",
"winapi", "winapi",
] ]
[[package]] [[package]]
name = "unic-langid" name = "unic-langid"
version = "0.9.3" version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "887622f8e7b723780c5e64b04dcc0c9b8f426ada7cca6790cd3ea3bf0f08037a" checksum = "238722e6d794ed130f91f4ea33e01fcff4f188d92337a21297892521c72df516"
dependencies = [ dependencies = [
"unic-langid-impl", "unic-langid-impl",
] ]
[[package]] [[package]]
name = "unic-langid-impl" name = "unic-langid-impl"
version = "0.9.3" version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5adeb847e35eed4efbffd9fb2e4d078b91ece56e4d6a3c0d2df55b3a1dac07d5" checksum = "4bd55a2063fdea4ef1f8633243a7b0524cbeef1905ae04c31a1c9b9775c55bc6"
dependencies = [ dependencies = [
"serde", "serde",
"tinystr", "tinystr",
@ -6348,7 +6349,7 @@ checksum = "6b2b8eecdb8e4284adf5546fc518f048f6dc33e7203dbe36fa93a4add39b31f6"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]
@ -6423,7 +6424,7 @@ dependencies = [
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -6457,7 +6458,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -6906,7 +6907,7 @@ dependencies = [
"js-sys", "js-sys",
"libc", "libc",
"log", "log",
"memmap2 0.9.0", "memmap2 0.9.1",
"ndk", "ndk",
"ndk-sys", "ndk-sys",
"objc2", "objc2",
@ -6936,9 +6937,9 @@ dependencies = [
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.5.26" version = "0.5.28"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b67b5f0a4e7a27a64c651977932b9dc5667ca7fc31ac44b03ed37a0cf42fdfff" checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@ -7000,21 +7001,20 @@ dependencies = [
[[package]] [[package]]
name = "xattr" name = "xattr"
version = "1.1.1" version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbc6ab6ec1907d1a901cdbcd2bd4cb9e7d64ce5c9739cbb97d3c391acd8c7fae" checksum = "a7dae5072fe1f8db8f8d29059189ac175196e410e40ba42d5d4684ae2f750995"
dependencies = [ dependencies = [
"libc", "libc",
"linux-raw-sys 0.4.12",
"rustix 0.38.28",
] ]
[[package]] [[package]]
name = "xcursor" name = "xcursor"
version = "0.3.4" version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "463705a63313cd4301184381c5e8042f0a7e9b4bb63653f216311d4ae74690b7" checksum = "6a0ccd7b4a5345edfcd0c3535718a4e9ff7798ffc536bb5b5a0e26ff84732911"
dependencies = [
"nom",
]
[[package]] [[package]]
name = "xdg-home" name = "xdg-home"
@ -7157,22 +7157,22 @@ dependencies = [
[[package]] [[package]]
name = "zerocopy" name = "zerocopy"
version = "0.7.30" version = "0.7.31"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d"
dependencies = [ dependencies = [
"zerocopy-derive", "zerocopy-derive",
] ]
[[package]] [[package]]
name = "zerocopy-derive" name = "zerocopy-derive"
version = "0.7.30" version = "0.7.31"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn 2.0.39", "syn 2.0.41",
] ]
[[package]] [[package]]

4
Changelog.md
View file

@ -5,7 +5,7 @@
### CLI ### CLI
- Providing full static rust binary with [Eyra](https://github.com/sunfishcode/eyra) - [#1102](https://github.com/qarmin/czkawka/pull/1102) - Providing full static rust binary with [Eyra](https://github.com/sunfishcode/eyra) - [#1102](https://github.com/qarmin/czkawka/pull/1102)
- Fixed duplicated `-c` argument, now saving as compact json is handled via `-C` - ???? - Fixed duplicated `-c` argument, now saving as compact json is handled via `-C` - [#1153](https://github.com/qarmin/czkawka/pull/1153)
### Krokiet GUI ### Krokiet GUI
- Initial release of new gui - [#1102](https://github.com/qarmin/czkawka/pull/1102) - Initial release of new gui - [#1102](https://github.com/qarmin/czkawka/pull/1102)
@ -21,6 +21,8 @@
- Big speedup when searching for empty folders(especially with multithreading + cached FS schema) - [#1152](https://github.com/qarmin/czkawka/pull/1152) - Big speedup when searching for empty folders(especially with multithreading + cached FS schema) - [#1152](https://github.com/qarmin/czkawka/pull/1152)
- Collecting files for scan can be a lot of faster due lazy file metadata gathering - [#1152](https://github.com/qarmin/czkawka/pull/1152) - Collecting files for scan can be a lot of faster due lazy file metadata gathering - [#1152](https://github.com/qarmin/czkawka/pull/1152)
- Fixed recognizing not accessible folders as non-empty - [#1152](https://github.com/qarmin/czkawka/pull/1152) - Fixed recognizing not accessible folders as non-empty - [#1152](https://github.com/qarmin/czkawka/pull/1152)
- Unifying code for collecting files to scan - [#1159](https://github.com/qarmin/czkawka/pull/1159)
- Decrease memory usage when collecting files by removing unused fields in custom file entries structs - [#1159](https://github.com/qarmin/czkawka/pull/1159)
## Version 6.1.0 - 15.10.2023r ## Version 6.1.0 - 15.10.2023r
- BREAKING CHANGE - Changed cache saving method, deduplicated, optimized and simplified procedure(all files needs to be hashed again) - [#1072](https://github.com/qarmin/czkawka/pull/1072), [#1086](https://github.com/qarmin/czkawka/pull/1086) - BREAKING CHANGE - Changed cache saving method, deduplicated, optimized and simplified procedure(all files needs to be hashed again) - [#1072](https://github.com/qarmin/czkawka/pull/1072), [#1086](https://github.com/qarmin/czkawka/pull/1086)

9
czkawka_core/src/bad_extensions.rs
View file

@ -210,17 +210,10 @@ impl BadExtensions {
#[fun_time(message = "check_files", level = "debug")] #[fun_time(message = "check_files", level = "debug")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .common_data(&self.common_data)
.group_by(|_fe| ()) .group_by(|_fe| ())
.stop_receiver(stop_receiver) .stop_receiver(stop_receiver)
.progress_sender(progress_sender) .progress_sender(progress_sender)
.minimal_file_size(self.common_data.minimal_file_size)
.maximal_file_size(self.common_data.maximal_file_size)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.tool_type(self.common_data.tool_type)
.build() .build()
.run(); .run();

159
czkawka_core/src/big_file.rs
View file

@ -1,30 +1,16 @@
use std::collections::BTreeMap;
use std::fs; use std::fs;
use std::fs::DirEntry;
use std::io::Write; use std::io::Write;
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use crossbeam_channel::{Receiver, Sender}; use crossbeam_channel::{Receiver, Sender};
use fun_time::fun_time; use fun_time::fun_time;
use humansize::{format_size, BINARY}; use humansize::{format_size, BINARY};
use log::debug; use log::debug;
use rayon::prelude::*; use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use crate::common::{check_folder_children, check_if_stop_received, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, split_path_compare}; use crate::common_dir_traversal::{DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_dir_traversal::{common_read_dir, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::{DebugPrint, PrintResults}; use crate::common_traits::{DebugPrint, PrintResults};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileEntry {
pub path: PathBuf,
pub size: u64,
pub modified_date: u64,
}
#[derive(Copy, Clone, Eq, PartialEq)] #[derive(Copy, Clone, Eq, PartialEq)]
pub enum SearchMode { pub enum SearchMode {
BiggestFiles, BiggestFiles,
@ -66,134 +52,39 @@ impl BigFile {
self.debug_print(); self.debug_print();
} }
#[fun_time(message = "look_for_big_files", level = "debug")] // #[fun_time(message = "look_for_big_files", level = "debug")]
fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool { fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
let mut old_map: BTreeMap<u64, Vec<FileEntry>> = Default::default(); let result = DirTraversalBuilder::new()
.group_by(|_fe| ())
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.common_data(&self.common_data)
.minimal_file_size(1)
.max_stage(0)
.build()
.run();
let mut folders_to_check: Vec<PathBuf> = self.common_data.directories.included_directories.clone(); match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
let mut all_files = grouped_file_entries.into_values().flatten().collect::<Vec<_>>();
all_files.par_sort_unstable_by_key(|fe| fe.size);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = if self.search_mode == SearchMode::BiggestFiles {
prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None, self.common_data.tool_type); all_files.reverse();
}
debug!("Starting to search for big files"); if all_files.len() > self.number_of_files_to_check {
while !folders_to_check.is_empty() { all_files.truncate(self.number_of_files_to_check);
if check_if_stop_received(stop_receiver) { }
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let segments: Vec<_> = folders_to_check self.big_files = all_files;
.into_par_iter()
.map(|current_folder| {
let mut dir_result = vec![];
let mut warnings = vec![];
let mut fe_result = vec![];
let Some(read_dir) = common_read_dir(&current_folder, &mut warnings) else {
return (dir_result, warnings, fe_result);
};
// Check every sub folder/file/link etc.
for entry in read_dir {
let Ok(entry_data) = entry else {
continue;
};
let Ok(file_type) = entry_data.file_type() else {
continue;
};
if file_type.is_dir() {
check_folder_children(
&mut dir_result,
&mut warnings,
&entry_data,
self.common_data.recursive_search,
&self.common_data.directories,
&self.common_data.excluded_items,
);
} else if file_type.is_file() {
self.collect_file_entry(&atomic_counter, &entry_data, &mut fe_result, &mut warnings);
}
}
(dir_result, warnings, fe_result)
})
.collect();
let required_size = segments.iter().map(|(segment, _, _)| segment.len()).sum::<usize>();
folders_to_check = Vec::with_capacity(required_size);
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
for (size, fe) in fe_result { debug!("check_files - Found {} biggest/smallest files.", self.big_files.len());
old_map.entry(size).or_default().push(fe); true
}
} }
}
debug!("Collected {} files", old_map.len()); DirTraversalResult::Stopped => false,
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.extract_n_biggest_files(old_map);
true
}
pub fn collect_file_entry(&self, atomic_counter: &Arc<AtomicUsize>, entry_data: &DirEntry, fe_result: &mut Vec<(u64, FileEntry)>, warnings: &mut Vec<String>) {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if !self.common_data.allowed_extensions.check_if_entry_ends_with_extension(entry_data) {
return;
}
let current_file_name = entry_data.path();
if self.common_data.excluded_items.is_excluded(&current_file_name) {
return;
}
let Ok(metadata) = entry_data.metadata() else {
return;
};
if metadata.len() == 0 {
return;
}
let fe: FileEntry = FileEntry {
modified_date: get_modified_time(&metadata, warnings, &current_file_name, false),
path: current_file_name,
size: metadata.len(),
};
fe_result.push((fe.size, fe));
}
#[fun_time(message = "extract_n_biggest_files", level = "debug")]
pub fn extract_n_biggest_files(&mut self, old_map: BTreeMap<u64, Vec<FileEntry>>) {
let iter: Box<dyn Iterator<Item = _>>;
if self.search_mode == SearchMode::SmallestFiles {
iter = Box::new(old_map.into_iter());
} else {
iter = Box::new(old_map.into_iter().rev());
}
for (_size, mut vector) in iter {
if self.information.number_of_real_files < self.number_of_files_to_check {
if vector.len() > 1 {
vector.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
}
for file in vector {
if self.information.number_of_real_files < self.number_of_files_to_check {
self.big_files.push(file);
self.information.number_of_real_files += 1;
} else {
break;
}
}
} else {
break;
}
} }
} }

263
czkawka_core/src/broken_files.rs
View file

@ -1,10 +1,9 @@
use std::collections::BTreeMap; use std::collections::{BTreeMap, HashSet};
use std::fs::{DirEntry, File}; use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::{fs, mem, panic}; use std::{fs, mem, panic};
use crossbeam_channel::{Receiver, Sender}; use crossbeam_channel::{Receiver, Sender};
@ -18,23 +17,23 @@ use rayon::prelude::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::common::{ use crate::common::{
check_folder_children, check_if_stop_received, create_crash_message, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS, check_if_stop_received, create_crash_message, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS,
IMAGE_RS_BROKEN_FILES_EXTENSIONS, PDF_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS, IMAGE_RS_BROKEN_FILES_EXTENSIONS, PDF_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS,
}; };
use crate::common_cache::{get_broken_files_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_cache::{get_broken_files_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
use crate::common_dir_traversal::{common_read_dir, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::*; use crate::common_traits::*;
#[derive(Clone, Serialize, Deserialize, Debug)] #[derive(Clone, Serialize, Deserialize, Debug)]
pub struct FileEntry { pub struct BrokenEntry {
pub path: PathBuf, pub path: PathBuf,
pub modified_date: u64, pub modified_date: u64,
pub size: u64, pub size: u64,
pub type_of_file: TypeOfFile, pub type_of_file: TypeOfFile,
pub error_string: String, pub error_string: String,
} }
impl ResultEntry for FileEntry { impl ResultEntry for BrokenEntry {
fn get_path(&self) -> &Path { fn get_path(&self) -> &Path {
&self.path &self.path
} }
@ -46,6 +45,19 @@ impl ResultEntry for FileEntry {
} }
} }
impl FileEntry {
fn into_broken_entry(self) -> BrokenEntry {
BrokenEntry {
size: self.size,
path: self.path,
modified_date: self.modified_date,
type_of_file: TypeOfFile::Unknown,
error_string: String::new(),
}
}
}
#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Debug)] #[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Debug)]
pub enum TypeOfFile { pub enum TypeOfFile {
Unknown = -1, Unknown = -1,
@ -55,6 +67,8 @@ pub enum TypeOfFile {
PDF, PDF,
} }
const MAX_BROKEN_FILES_STAGE: u8 = 1;
bitflags! { bitflags! {
#[derive(PartialEq, Copy, Clone)] #[derive(PartialEq, Copy, Clone)]
pub struct CheckedTypes : u32 { pub struct CheckedTypes : u32 {
@ -75,8 +89,8 @@ pub struct Info {
pub struct BrokenFiles { pub struct BrokenFiles {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
files_to_check: BTreeMap<String, FileEntry>, files_to_check: BTreeMap<String, BrokenEntry>,
broken_files: Vec<FileEntry>, broken_files: Vec<BrokenEntry>,
checked_types: CheckedTypes, checked_types: CheckedTypes,
} }
@ -108,111 +122,62 @@ impl BrokenFiles {
#[fun_time(message = "check_files", level = "debug")] #[fun_time(message = "check_files", level = "debug")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
let mut folders_to_check: Vec<PathBuf> = self.common_data.directories.included_directories.clone(); let zip_extensions = ZIP_FILES_EXTENSIONS.iter().collect::<HashSet<_>>();
let audio_extensions = AUDIO_FILES_EXTENSIONS.iter().collect::<HashSet<_>>();
let pdf_extensions = PDF_FILES_EXTENSIONS.iter().collect::<HashSet<_>>();
let images_extensions = IMAGE_RS_BROKEN_FILES_EXTENSIONS.iter().collect::<HashSet<_>>();
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = let mut extensions = Vec::new();
prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None, self.common_data.tool_type); let vec_extensions = [
(CheckedTypes::PDF, PDF_FILES_EXTENSIONS),
debug!("check_files - starting to collect files"); (CheckedTypes::AUDIO, AUDIO_FILES_EXTENSIONS),
while !folders_to_check.is_empty() { (CheckedTypes::ARCHIVE, ZIP_FILES_EXTENSIONS),
if check_if_stop_received(stop_receiver) { (CheckedTypes::IMAGE, IMAGE_RS_BROKEN_FILES_EXTENSIONS),
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); ];
return false; for (checked_type, extensions_to_add) in &vec_extensions {
if self.checked_types.contains(*checked_type) {
extensions.extend_from_slice(extensions_to_add);
} }
}
let segments: Vec<_> = folders_to_check self.common_data.allowed_extensions.set_and_validate_extensions(&extensions);
.into_par_iter() if !self.common_data.allowed_extensions.set_any_extensions() {
.map(|current_folder| { return true;
let mut dir_result = vec![]; }
let mut warnings = vec![];
let mut fe_result = vec![];
let Some(read_dir) = common_read_dir(&current_folder, &mut warnings) else { let result = DirTraversalBuilder::new()
return (dir_result, warnings, fe_result); .group_by(|_fe| ())
}; .stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.common_data(&self.common_data)
.max_stage(MAX_BROKEN_FILES_STAGE)
.build()
.run();
// Check every sub folder/file/link etc. match result {
for entry in read_dir { DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
let Ok(entry_data) = entry else { self.broken_files = grouped_file_entries
continue; .into_values()
}; .flatten()
let Ok(file_type) = entry_data.file_type() else { .map(|fe| {
continue; let mut broken_entry = fe.into_broken_entry();
}; broken_entry.type_of_file = check_extension_availability(broken_entry.get_path(), &images_extensions, &zip_extensions, &audio_extensions, &pdf_extensions);
broken_entry
if file_type.is_dir() { })
check_folder_children( .collect();
&mut dir_result,
&mut warnings,
&entry_data,
self.common_data.recursive_search,
&self.common_data.directories,
&self.common_data.excluded_items,
);
} else if file_type.is_file() {
if let Some(file_entry) = self.get_file_entry(&atomic_counter, &entry_data, &mut warnings) {
fe_result.push((file_entry.path.to_string_lossy().to_string(), file_entry));
}
}
}
(dir_result, warnings, fe_result)
})
.collect();
debug!("check_files - collected files");
let required_size = segments.iter().map(|(segment, _, _)| segment.len()).sum::<usize>();
folders_to_check = Vec::with_capacity(required_size);
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
for (name, fe) in fe_result { debug!("check_files - Found {} image files.", self.broken_files.len());
self.files_to_check.insert(name, fe); true
}
} }
}
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); DirTraversalResult::Stopped => false,
true }
} }
fn get_file_entry(&self, atomic_counter: &Arc<AtomicUsize>, entry_data: &DirEntry, warnings: &mut Vec<String>) -> Option<FileEntry> { fn check_broken_image(&self, mut file_entry: BrokenEntry) -> Option<BrokenEntry> {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if !self.common_data.allowed_extensions.check_if_entry_ends_with_extension(entry_data) {
return None;
}
let file_name_lowercase = entry_data.file_name().to_string_lossy().to_lowercase();
let type_of_file = check_extension_availability(&file_name_lowercase);
if !check_if_file_extension_is_allowed(&type_of_file, &self.checked_types) {
return None;
}
let current_file_name = entry_data.path();
if self.common_data.excluded_items.is_excluded(&current_file_name) {
return None;
}
let Ok(metadata) = entry_data.metadata() else {
return None;
};
let fe: FileEntry = FileEntry {
modified_date: get_modified_time(&metadata, warnings, &current_file_name, false),
path: current_file_name,
size: metadata.len(),
type_of_file,
error_string: String::new(),
};
Some(fe)
}
fn check_broken_image(&self, mut file_entry: FileEntry) -> Option<FileEntry> {
let mut file_entry_clone = file_entry.clone(); let mut file_entry_clone = file_entry.clone();
let result = panic::catch_unwind(|| { panic::catch_unwind(|| {
if let Err(e) = image::open(&file_entry.path) { if let Err(e) = image::open(&file_entry.path) {
let error_string = e.to_string(); let error_string = e.to_string();
// This error is a problem with image library, remove check when https://github.com/image-rs/jpeg-decoder/issues/130 will be fixed // This error is a problem with image library, remove check when https://github.com/image-rs/jpeg-decoder/issues/130 will be fixed
@ -222,19 +187,15 @@ impl BrokenFiles {
file_entry.error_string = error_string; file_entry.error_string = error_string;
} }
Some(file_entry) Some(file_entry)
}); })
.unwrap_or_else(|_| {
// If image crashed during opening, needs to be printed info about crashes thing
if let Ok(image_result) = result {
image_result
} else {
let message = create_crash_message("Image-rs", &file_entry_clone.path.to_string_lossy(), "https://github.com/Serial-ATA/lofty-rs"); let message = create_crash_message("Image-rs", &file_entry_clone.path.to_string_lossy(), "https://github.com/Serial-ATA/lofty-rs");
println!("{message}"); println!("{message}");
file_entry_clone.error_string = message; file_entry_clone.error_string = message;
Some(file_entry_clone) Some(file_entry_clone)
} })
} }
fn check_broken_zip(&self, mut file_entry: FileEntry) -> Option<FileEntry> { fn check_broken_zip(&self, mut file_entry: BrokenEntry) -> Option<BrokenEntry> {
match File::open(&file_entry.path) { match File::open(&file_entry.path) {
Ok(file) => { Ok(file) => {
if let Err(e) = zip::ZipArchive::new(file) { if let Err(e) = zip::ZipArchive::new(file) {
@ -245,35 +206,32 @@ impl BrokenFiles {
Err(_inspected) => None, Err(_inspected) => None,
} }
} }
fn check_broken_audio(&self, mut file_entry: FileEntry) -> Option<FileEntry> { fn check_broken_audio(&self, mut file_entry: BrokenEntry) -> Option<BrokenEntry> {
match File::open(&file_entry.path) { match File::open(&file_entry.path) {
Ok(file) => { Ok(file) => {
let mut file_entry_clone = file_entry.clone(); let mut file_entry_clone = file_entry.clone();
let result = panic::catch_unwind(|| { panic::catch_unwind(|| {
if let Err(e) = audio_checker::parse_audio_file(file) { if let Err(e) = audio_checker::parse_audio_file(file) {
file_entry.error_string = e.to_string(); file_entry.error_string = e.to_string();
} }
Some(file_entry) Some(file_entry)
}); })
.unwrap_or_else(|_| {
if let Ok(audio_result) = result {
audio_result
} else {
let message = create_crash_message("Symphonia", &file_entry_clone.path.to_string_lossy(), "https://github.com/pdeljanov/Symphonia"); let message = create_crash_message("Symphonia", &file_entry_clone.path.to_string_lossy(), "https://github.com/pdeljanov/Symphonia");
println!("{message}"); println!("{message}");
file_entry_clone.error_string = message; file_entry_clone.error_string = message;
Some(file_entry_clone) Some(file_entry_clone)
} })
} }
Err(_inspected) => None, Err(_inspected) => None,
} }
} }
fn check_broken_pdf(&self, mut file_entry: FileEntry) -> Option<FileEntry> { fn check_broken_pdf(&self, mut file_entry: BrokenEntry) -> Option<BrokenEntry> {
let parser_options = ParseOptions::tolerant(); // Only show as broken files with really big bugs let parser_options = ParseOptions::tolerant(); // Only show as broken files with really big bugs
let mut file_entry_clone = file_entry.clone(); let mut file_entry_clone = file_entry.clone();
let result = panic::catch_unwind(|| { panic::catch_unwind(|| {
match FileOptions::cached().parse_options(parser_options).open(&file_entry.path) { match FileOptions::cached().parse_options(parser_options).open(&file_entry.path) {
Ok(file) => { Ok(file) => {
for idx in 0..file.num_pages() { for idx in 0..file.num_pages() {
@ -297,27 +255,26 @@ impl BrokenFiles {
} }
} }
Some(file_entry) Some(file_entry)
}); })
if let Ok(pdf_result) = result { .unwrap_or_else(|_| {
pdf_result
} else {
let message = create_crash_message("PDF-rs", &file_entry_clone.path.to_string_lossy(), "https://github.com/pdf-rs/pdf"); let message = create_crash_message("PDF-rs", &file_entry_clone.path.to_string_lossy(), "https://github.com/pdf-rs/pdf");
println!("{message}"); println!("{message}");
file_entry_clone.error_string = message; file_entry_clone.error_string = message;
Some(file_entry_clone) Some(file_entry_clone)
} })
} }
#[fun_time(message = "load_cache", level = "debug")] #[fun_time(message = "load_cache", level = "debug")]
fn load_cache(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) { fn load_cache(&mut self) -> (BTreeMap<String, BrokenEntry>, BTreeMap<String, BrokenEntry>, BTreeMap<String, BrokenEntry>) {
let loaded_hash_map; let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default(); let mut records_already_cached: BTreeMap<String, BrokenEntry> = Default::default();
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default(); let mut non_cached_files_to_check: BTreeMap<String, BrokenEntry> = Default::default();
let files_to_check = mem::take(&mut self.files_to_check); let files_to_check = mem::take(&mut self.files_to_check);
if self.common_data.use_cache { if self.common_data.use_cache {
let (messages, loaded_items) = load_cache_from_file_generalized_by_path::<FileEntry>(&get_broken_files_cache_file(), self.get_delete_outdated_cache(), &files_to_check); let (messages, loaded_items) =
load_cache_from_file_generalized_by_path::<BrokenEntry>(&get_broken_files_cache_file(), self.get_delete_outdated_cache(), &files_to_check);
self.get_text_messages_mut().extend_with_another_messages(messages); self.get_text_messages_mut().extend_with_another_messages(messages);
loaded_hash_map = loaded_items.unwrap_or_default(); loaded_hash_map = loaded_items.unwrap_or_default();
@ -343,7 +300,7 @@ impl BrokenFiles {
prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type);
debug!("look_for_broken_files - started finding for broken files"); debug!("look_for_broken_files - started finding for broken files");
let mut vec_file_entry: Vec<FileEntry> = non_cached_files_to_check let mut vec_file_entry: Vec<BrokenEntry> = non_cached_files_to_check
.into_par_iter() .into_par_iter()
.map(|(_, file_entry)| { .map(|(_, file_entry)| {
atomic_counter.fetch_add(1, Ordering::Relaxed); atomic_counter.fetch_add(1, Ordering::Relaxed);
@ -363,7 +320,7 @@ impl BrokenFiles {
.while_some() .while_some()
.filter(Option::is_some) .filter(Option::is_some)
.map(Option::unwrap) .map(Option::unwrap)
.collect::<Vec<FileEntry>>(); .collect::<Vec<BrokenEntry>>();
debug!("look_for_broken_files - ended finding for broken files"); debug!("look_for_broken_files - ended finding for broken files");
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
@ -386,10 +343,10 @@ impl BrokenFiles {
true true
} }
#[fun_time(message = "save_to_cache", level = "debug")] #[fun_time(message = "save_to_cache", level = "debug")]
fn save_to_cache(&mut self, vec_file_entry: &[FileEntry], loaded_hash_map: BTreeMap<String, FileEntry>) { fn save_to_cache(&mut self, vec_file_entry: &[BrokenEntry], loaded_hash_map: BTreeMap<String, BrokenEntry>) {
if self.common_data.use_cache { if self.common_data.use_cache {
// Must save all results to file, old loaded from file with all currently counted results // Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = Default::default(); let mut all_results: BTreeMap<String, BrokenEntry> = Default::default();
for file_entry in vec_file_entry.iter().cloned() { for file_entry in vec_file_entry.iter().cloned() {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry); all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
@ -424,7 +381,7 @@ impl BrokenFiles {
} }
impl BrokenFiles { impl BrokenFiles {
pub const fn get_broken_files(&self) -> &Vec<FileEntry> { pub const fn get_broken_files(&self) -> &Vec<BrokenEntry> {
&self.broken_files &self.broken_files
} }
@ -478,27 +435,37 @@ impl PrintResults for BrokenFiles {
} }
} }
fn check_extension_availability(file_name_lowercase: &str) -> TypeOfFile { fn check_extension_availability(
if IMAGE_RS_BROKEN_FILES_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) { full_name: &Path,
images_extensions: &HashSet<&&'static str>,
zip_extensions: &HashSet<&&'static str>,
audio_extensions: &HashSet<&&'static str>,
pdf_extensions: &HashSet<&&'static str>,
) -> TypeOfFile {
let Some(extension) = full_name.extension() else {
debug_assert!(false, "Missing extension");
return TypeOfFile::Unknown;
};
let Some(extension_str) = extension.to_str() else {
debug_assert!(false, "Extension not really fully str");
return TypeOfFile::Unknown;
};
if images_extensions.contains(&extension_str) {
TypeOfFile::Image TypeOfFile::Image
} else if ZIP_FILES_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) { } else if zip_extensions.contains(&extension_str) {
TypeOfFile::ArchiveZip TypeOfFile::ArchiveZip
} else if AUDIO_FILES_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) { } else if audio_extensions.contains(&extension_str) {
TypeOfFile::Audio TypeOfFile::Audio
} else if PDF_FILES_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) { } else if pdf_extensions.contains(&extension_str) {
TypeOfFile::PDF TypeOfFile::PDF
} else { } else {
debug_assert!(false, "File with unknown extension");
TypeOfFile::Unknown TypeOfFile::Unknown
} }
} }
fn check_if_file_extension_is_allowed(type_of_file: &TypeOfFile, checked_types: &CheckedTypes) -> bool {
((*type_of_file == TypeOfFile::Image) && ((*checked_types & CheckedTypes::IMAGE) == CheckedTypes::IMAGE))
|| ((*type_of_file == TypeOfFile::PDF) && ((*checked_types & CheckedTypes::PDF) == CheckedTypes::PDF))
|| ((*type_of_file == TypeOfFile::ArchiveZip) && ((*checked_types & CheckedTypes::ARCHIVE) == CheckedTypes::ARCHIVE))
|| ((*type_of_file == TypeOfFile::Audio) && ((*checked_types & CheckedTypes::AUDIO) == CheckedTypes::AUDIO))
}
fn unpack_pdf_error(e: PdfError) -> PdfError { fn unpack_pdf_error(e: PdfError) -> PdfError {
if let Try { if let Try {
file: _, file: _,
@ -514,7 +481,7 @@ fn unpack_pdf_error(e: PdfError) -> PdfError {
} }
} }
fn validate_pdf_error(file_entry: &mut FileEntry, e: PdfError) -> PdfError { fn validate_pdf_error(file_entry: &mut BrokenEntry, e: PdfError) -> PdfError {
let mut error_string = e.to_string(); let mut error_string = e.to_string();
// Workaround for strange error message https://github.com/qarmin/czkawka/issues/898 // Workaround for strange error message https://github.com/qarmin/czkawka/issues/898
if error_string.starts_with("Try at") { if error_string.starts_with("Try at") {

21
czkawka_core/src/common.rs
View file

@ -123,30 +123,27 @@ pub fn set_number_of_threads(thread_number: usize) {
} }
pub const RAW_IMAGE_EXTENSIONS: &[&str] = &[ pub const RAW_IMAGE_EXTENSIONS: &[&str] = &[
".mrw", ".arw", ".srf", ".sr2", ".mef", ".orf", ".srw", ".erf", ".kdc", ".kdc", ".dcs", ".rw2", ".raf", ".dcr", ".dng", ".pef", ".crw", ".iiq", ".3fr", ".nrw", ".nef", ".mos", "mrw", "arw", "srf", "sr2", "mef", "orf", "srw", "erf", "kdc", "kdc", "dcs", "rw2", "raf", "dcr", "dng", "pef", "crw", "iiq", "3fr", "nrw", "nef", "mos", "cr2", "ari",
".cr2", ".ari",
];
pub const IMAGE_RS_EXTENSIONS: &[&str] = &[
".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif", ".tga", ".ff", ".jif", ".jfi", ".webp", ".gif", ".ico", ".exr", ".qoi",
]; ];
pub const IMAGE_RS_EXTENSIONS: &[&str] = &["jpg", "jpeg", "png", "bmp", "tiff", "tif", "tga", "ff", "jif", "jfi", "webp", "gif", "ico", "exr", "qoi"];
pub const IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS: &[&str] = &[".jpg", ".jpeg", ".png", ".tiff", ".tif", ".tga", ".ff", ".jif", ".jfi", ".bmp", ".webp", ".exr", ".qoi"]; pub const IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS: &[&str] = &["jpg", "jpeg", "png", "tiff", "tif", "tga", "ff", "jif", "jfi", "bmp", "webp", "exr", "qoi"];
pub const IMAGE_RS_BROKEN_FILES_EXTENSIONS: &[&str] = &[ pub const IMAGE_RS_BROKEN_FILES_EXTENSIONS: &[&str] = &[
".jpg", ".jpeg", ".png", ".tiff", ".tif", ".tga", ".ff", ".jif", ".jfi", ".gif", ".bmp", ".ico", ".jfif", ".jpe", ".pnz", ".dib", ".webp", ".exr", "jpg", "jpeg", "png", "tiff", "tif", "tga", "ff", "jif", "jfi", "gif", "bmp", "ico", "jfif", "jpe", "pnz", "dib", "webp", "exr",
]; ];
pub const HEIC_EXTENSIONS: &[&str] = &[".heif", ".heifs", ".heic", ".heics", ".avci", ".avcs", ".avifs"]; pub const HEIC_EXTENSIONS: &[&str] = &["heif", "heifs", "heic", "heics", "avci", "avcs", "avifs"];
pub const ZIP_FILES_EXTENSIONS: &[&str] = &[".zip", ".jar"]; pub const ZIP_FILES_EXTENSIONS: &[&str] = &["zip", "jar"];
pub const PDF_FILES_EXTENSIONS: &[&str] = &[".pdf"]; pub const PDF_FILES_EXTENSIONS: &[&str] = &["pdf"];
pub const AUDIO_FILES_EXTENSIONS: &[&str] = &[ pub const AUDIO_FILES_EXTENSIONS: &[&str] = &[
".mp3", ".flac", ".wav", ".ogg", ".m4a", ".aac", ".aiff", ".pcm", ".aif", ".aiff", ".aifc", ".m3a", ".mp2", ".mp4a", ".mp2a", ".mpga", ".wave", ".weba", ".wma", ".oga", "mp3", "flac", "wav", "ogg", "m4a", "aac", "aiff", "pcm", "aif", "aiff", "aifc", "m3a", "mp2", "mp4a", "mp2a", "mpga", "wave", "weba", "wma", "oga",
]; ];
pub const VIDEO_FILES_EXTENSIONS: &[&str] = &[ pub const VIDEO_FILES_EXTENSIONS: &[&str] = &[
".mp4", ".mpv", ".flv", ".mp4a", ".webm", ".mpg", ".mp2", ".mpeg", ".m4p", ".m4v", ".avi", ".wmv", ".qt", ".mov", ".swf", ".mkv", "mp4", "mpv", "flv", "mp4a", "webm", "mpg", "mp2", "mpeg", "m4p", "m4v", "avi", "wmv", "qt", "mov", "swf", "mkv",
]; ];
pub const LOOP_DURATION: u32 = 20; //ms pub const LOOP_DURATION: u32 = 20; //ms

10
czkawka_core/src/common_cache.rs
View file

@ -203,11 +203,9 @@ where
vec_loaded_entries = vec_loaded_entries vec_loaded_entries = vec_loaded_entries
.into_par_iter() .into_par_iter()
.filter(|file_entry| { .filter(|file_entry| {
if delete_outdated_cache && !file_entry.get_path().exists() { let path = file_entry.get_path();
return false;
}
let file_entry_path_str = file_entry.get_path().to_string_lossy().to_string(); let file_entry_path_str = path.to_string_lossy().to_string();
if let Some(used_file) = used_files.get(&file_entry_path_str) { if let Some(used_file) = used_files.get(&file_entry_path_str) {
if file_entry.get_size() != used_file.get_size() { if file_entry.get_size() != used_file.get_size() {
return false; return false;
@ -217,6 +215,10 @@ where
} }
} }
if delete_outdated_cache && !path.exists() {
return false;
}
true true
}) })
.collect(); .collect();

80
czkawka_core/src/common_dir_traversal.rs
View file

@ -15,6 +15,7 @@ use crate::common::{check_if_stop_received, prepare_thread_handler_common, send_
use crate::common_directory::Directories; use crate::common_directory::Directories;
use crate::common_extensions::Extensions; use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems; use crate::common_items::ExcludedItems;
use crate::common_tool::CommonToolData;
use crate::common_traits::ResultEntry; use crate::common_traits::ResultEntry;
use crate::flc; use crate::flc;
use crate::localizer_core::generate_translation_hashmap; use crate::localizer_core::generate_translation_hashmap;
@ -63,8 +64,6 @@ pub struct FileEntry {
pub path: PathBuf, pub path: PathBuf,
pub size: u64, pub size: u64,
pub modified_date: u64, pub modified_date: u64,
pub hash: String,
pub symlink_info: Option<SymlinkInfo>,
} }
impl ResultEntry for FileEntry { impl ResultEntry for FileEntry {
@ -81,14 +80,6 @@ impl ResultEntry for FileEntry {
// Symlinks // Symlinks
const MAX_NUMBER_OF_SYMLINK_JUMPS: i32 = 20;
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub struct SymlinkInfo {
pub destination_path: PathBuf,
pub type_of_error: ErrorType,
}
#[derive(Clone, Debug, PartialEq, Eq, Copy, Deserialize, Serialize)] #[derive(Clone, Debug, PartialEq, Eq, Copy, Deserialize, Serialize)]
pub enum ErrorType { pub enum ErrorType {
InfiniteRecursion, InfiniteRecursion,
@ -176,6 +167,18 @@ impl<'a, 'b, F> DirTraversalBuilder<'a, 'b, F> {
self self
} }
pub fn common_data(mut self, common_tool_data: &CommonToolData) -> Self {
self.root_dirs = common_tool_data.directories.included_directories.clone();
self.allowed_extensions = Some(common_tool_data.allowed_extensions.clone());
self.excluded_items = Some(common_tool_data.excluded_items.clone());
self.recursive_search = common_tool_data.recursive_search;
self.minimal_file_size = Some(common_tool_data.minimal_file_size);
self.maximal_file_size = Some(common_tool_data.maximal_file_size);
self.tool_type = common_tool_data.tool_type;
self.directories = Some(common_tool_data.directories.clone());
self
}
pub fn stop_receiver(mut self, stop_receiver: Option<&'a Receiver<()>>) -> Self { pub fn stop_receiver(mut self, stop_receiver: Option<&'a Receiver<()>>) -> Self {
self.stop_receiver = stop_receiver; self.stop_receiver = stop_receiver;
self self
@ -464,8 +467,6 @@ fn process_file_in_file_mode(
size: metadata.len(), size: metadata.len(),
modified_date: get_modified_time(&metadata, warnings, &current_file_name, false), modified_date: get_modified_time(&metadata, warnings, &current_file_name, false),
path: current_file_name, path: current_file_name,
hash: String::new(),
symlink_info: None,
}; };
fe_result.push(fe); fe_result.push(fe);
@ -535,51 +536,13 @@ fn process_symlink_in_symlink_mode(
return; return;
}; };
let mut destination_path = PathBuf::new();
let type_of_error;
match current_file_name.read_link() {
Ok(t) => {
destination_path.push(t);
let mut number_of_loop = 0;
let mut current_path = current_file_name.clone();
loop {
if number_of_loop == 0 && !current_path.exists() {
type_of_error = ErrorType::NonExistentFile;
break;
}
if number_of_loop == MAX_NUMBER_OF_SYMLINK_JUMPS {
type_of_error = ErrorType::InfiniteRecursion;
break;
}
current_path = match current_path.read_link() {
Ok(t) => t,
Err(_inspected) => {
// Looks that some next symlinks are broken, but we do nothing with it - TODO why they are broken
return;
}
};
number_of_loop += 1;
}
}
Err(_inspected) => {
// Failed to load info about it
type_of_error = ErrorType::NonExistentFile;
}
}
// Creating new file entry // Creating new file entry
let fe: FileEntry = FileEntry { let fe: FileEntry = FileEntry {
size: metadata.len(),
modified_date: get_modified_time(&metadata, warnings, &current_file_name, false), modified_date: get_modified_time(&metadata, warnings, &current_file_name, false),
path: current_file_name, path: current_file_name,
size: 0,
hash: String::new(),
symlink_info: Some(SymlinkInfo { destination_path, type_of_error }),
}; };
// Adding files to Vector
fe_result.push(fe); fe_result.push(fe);
} }
@ -671,18 +634,3 @@ pub fn get_modified_time(metadata: &Metadata, warnings: &mut Vec<String>, curren
} }
} }
} }
pub fn get_lowercase_name(entry_data: &DirEntry, warnings: &mut Vec<String>) -> Option<String> {
let name = match entry_data.file_name().into_string() {
Ok(t) => t,
Err(_inspected) => {
warnings.push(flc!(
"core_file_not_utf8_name",
generate_translation_hashmap(vec![("name", entry_data.path().to_string_lossy().to_string())])
));
return None;
}
}
.to_lowercase();
Some(name)
}

1
czkawka_core/src/common_directory.rs
View file

@ -122,7 +122,6 @@ impl Directories {
} }
// Try to canonicalize them // Try to canonicalize them
if cfg!(windows) { if cfg!(windows) {
// Only canonicalize if it's not a network path // Only canonicalize if it's not a network path
// This can be check by checking if path starts with \\?\UNC\ // This can be check by checking if path starts with \\?\UNC\

22
czkawka_core/src/common_extensions.rs
View file

@ -79,14 +79,32 @@ impl Extensions {
} }
} }
pub fn using_custom_extensions(&self) -> bool { pub fn set_any_extensions(&self) -> bool {
!self.file_extensions_hashset.is_empty() !self.file_extensions_hashset.is_empty()
} }
pub fn extend_allowed_extensions(&mut self, file_extensions: &[&str]) { fn extend_allowed_extensions(&mut self, file_extensions: &[&str]) {
for extension in file_extensions { for extension in file_extensions {
let extension_without_dot = extension.trim_start_matches('.'); let extension_without_dot = extension.trim_start_matches('.');
self.file_extensions_hashset.insert(extension_without_dot.to_string()); self.file_extensions_hashset.insert(extension_without_dot.to_string());
} }
} }
// E.g. when using similar videos, user can provide extensions like "mp4,flv", but if user provide "mp4,jpg" then
// it will be only "mp4" because "jpg" is not valid extension for videos
fn union_allowed_extensions(&mut self, file_extensions: &[&str]) {
let mut new_extensions = HashSet::new();
for extension in file_extensions {
let extension_without_dot = extension.trim_start_matches('.');
new_extensions.insert(extension_without_dot.to_string());
}
}
pub fn set_and_validate_extensions(&mut self, file_extensions: &[&str]) {
if self.file_extensions_hashset.is_empty() {
self.extend_allowed_extensions(file_extensions);
} else {
self.union_allowed_extensions(file_extensions);
}
}
} }

198
czkawka_core/src/duplicate.rs
View file

@ -6,7 +6,7 @@ use std::io::prelude::*;
use std::io::{self, Error, ErrorKind}; use std::io::{self, Error, ErrorKind};
#[cfg(target_family = "unix")] #[cfg(target_family = "unix")]
use std::os::unix::fs::MetadataExt; use std::os::unix::fs::MetadataExt;
use std::path::Path; use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
use std::{fs, mem}; use std::{fs, mem};
@ -15,6 +15,7 @@ use fun_time::fun_time;
use humansize::{format_size, BINARY}; use humansize::{format_size, BINARY};
use log::debug; use log::debug;
use rayon::prelude::*; use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use xxhash_rust::xxh3::Xxh3; use xxhash_rust::xxh3::Xxh3;
use crate::common::{check_if_stop_received, delete_files_custom, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads}; use crate::common::{check_if_stop_received, delete_files_custom, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
@ -44,6 +45,36 @@ impl HashType {
} }
} }
#[derive(Clone, Serialize, Deserialize, Debug, Default)]
pub struct DuplicateEntry {
pub path: PathBuf,
pub modified_date: u64,
pub size: u64,
pub hash: String,
}
impl ResultEntry for DuplicateEntry {
fn get_path(&self) -> &Path {
&self.path
}
fn get_modified_date(&self) -> u64 {
self.modified_date
}
fn get_size(&self) -> u64 {
self.size
}
}
impl FileEntry {
fn into_duplicate_entry(self) -> DuplicateEntry {
DuplicateEntry {
size: self.size,
path: self.path,
modified_date: self.modified_date,
hash: String::new(),
}
}
}
#[derive(Default)] #[derive(Default)]
pub struct Info { pub struct Info {
pub number_of_groups_by_size: usize, pub number_of_groups_by_size: usize,
@ -62,21 +93,21 @@ pub struct DuplicateFinder {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
// File Size, File Entry // File Size, File Entry
files_with_identical_names: BTreeMap<String, Vec<FileEntry>>, files_with_identical_names: BTreeMap<String, Vec<DuplicateEntry>>,
// File (Size, Name), File Entry // File (Size, Name), File Entry
files_with_identical_size_names: BTreeMap<(u64, String), Vec<FileEntry>>, files_with_identical_size_names: BTreeMap<(u64, String), Vec<DuplicateEntry>>,
// File Size, File Entry // File Size, File Entry
files_with_identical_size: BTreeMap<u64, Vec<FileEntry>>, files_with_identical_size: BTreeMap<u64, Vec<DuplicateEntry>>,
// File Size, next grouped by file size, next grouped by hash // File Size, next grouped by file size, next grouped by hash
files_with_identical_hashes: BTreeMap<u64, Vec<Vec<FileEntry>>>, files_with_identical_hashes: BTreeMap<u64, Vec<Vec<DuplicateEntry>>>,
// File Size, File Entry // File Size, File Entry
files_with_identical_names_referenced: BTreeMap<String, (FileEntry, Vec<FileEntry>)>, files_with_identical_names_referenced: BTreeMap<String, (DuplicateEntry, Vec<DuplicateEntry>)>,
// File (Size, Name), File Entry // File (Size, Name), File Entry
files_with_identical_size_names_referenced: BTreeMap<(u64, String), (FileEntry, Vec<FileEntry>)>, files_with_identical_size_names_referenced: BTreeMap<(u64, String), (DuplicateEntry, Vec<DuplicateEntry>)>,
// File Size, File Entry // File Size, File Entry
files_with_identical_size_referenced: BTreeMap<u64, (FileEntry, Vec<FileEntry>)>, files_with_identical_size_referenced: BTreeMap<u64, (DuplicateEntry, Vec<DuplicateEntry>)>,
// File Size, next grouped by file size, next grouped by hash // File Size, next grouped by file size, next grouped by hash
files_with_identical_hashes_referenced: BTreeMap<u64, Vec<(FileEntry, Vec<FileEntry>)>>, files_with_identical_hashes_referenced: BTreeMap<u64, Vec<(DuplicateEntry, Vec<DuplicateEntry>)>>,
check_method: CheckingMethod, check_method: CheckingMethod,
hash_type: HashType, hash_type: HashType,
ignore_hard_links: bool, ignore_hard_links: bool,
@ -158,18 +189,11 @@ impl DuplicateFinder {
}; };
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .common_data(&self.common_data)
.group_by(group_by_func) .group_by(group_by_func)
.stop_receiver(stop_receiver) .stop_receiver(stop_receiver)
.progress_sender(progress_sender) .progress_sender(progress_sender)
.checking_method(CheckingMethod::Name) .checking_method(CheckingMethod::Name)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.minimal_file_size(self.common_data.minimal_file_size)
.maximal_file_size(self.common_data.maximal_file_size)
.tool_type(self.common_data.tool_type)
.build() .build()
.run(); .run();
@ -178,7 +202,16 @@ impl DuplicateFinder {
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
// Create new BTreeMap without single size entries(files have not duplicates) // Create new BTreeMap without single size entries(files have not duplicates)
self.files_with_identical_names = grouped_file_entries.into_iter().filter(|(_name, vector)| vector.len() > 1).collect(); self.files_with_identical_names = grouped_file_entries
.into_iter()
.filter_map(|(name, vector)| {
if vector.len() > 1 {
Some((name, vector.into_iter().map(FileEntry::into_duplicate_entry).collect()))
} else {
None
}
})
.collect();
// Reference - only use in size, because later hash will be counted differently // Reference - only use in size, because later hash will be counted differently
if self.common_data.use_reference_folders { if self.common_data.use_reference_folders {
@ -195,7 +228,7 @@ impl DuplicateFinder {
Some((files_from_referenced_folders.pop().unwrap(), normal_files)) Some((files_from_referenced_folders.pop().unwrap(), normal_files))
} }
}) })
.collect::<Vec<(FileEntry, Vec<FileEntry>)>>(); .collect::<Vec<(DuplicateEntry, Vec<DuplicateEntry>)>>();
for (fe, vec_fe) in vec { for (fe, vec_fe) in vec {
self.files_with_identical_names_referenced.insert(fe.path.to_string_lossy().to_string(), (fe, vec_fe)); self.files_with_identical_names_referenced.insert(fe.path.to_string_lossy().to_string(), (fe, vec_fe));
} }
@ -231,18 +264,11 @@ impl DuplicateFinder {
}; };
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .common_data(&self.common_data)
.group_by(group_by_func) .group_by(group_by_func)
.stop_receiver(stop_receiver) .stop_receiver(stop_receiver)
.progress_sender(progress_sender) .progress_sender(progress_sender)
.checking_method(CheckingMethod::Name) .checking_method(CheckingMethod::Name)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.minimal_file_size(self.common_data.minimal_file_size)
.maximal_file_size(self.common_data.maximal_file_size)
.tool_type(self.common_data.tool_type)
.build() .build()
.run(); .run();
@ -250,7 +276,16 @@ impl DuplicateFinder {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => { DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
self.files_with_identical_size_names = grouped_file_entries.into_iter().filter(|(_name, vector)| vector.len() > 1).collect(); self.files_with_identical_size_names = grouped_file_entries
.into_iter()
.filter_map(|(size_name, vector)| {
if vector.len() > 1 {
Some((size_name, vector.into_iter().map(FileEntry::into_duplicate_entry).collect()))
} else {
None
}
})
.collect();
// Reference - only use in size, because later hash will be counted differently // Reference - only use in size, because later hash will be counted differently
if self.common_data.use_reference_folders { if self.common_data.use_reference_folders {
@ -267,7 +302,7 @@ impl DuplicateFinder {
Some((files_from_referenced_folders.pop().unwrap(), normal_files)) Some((files_from_referenced_folders.pop().unwrap(), normal_files))
} }
}) })
.collect::<Vec<(FileEntry, Vec<FileEntry>)>>(); .collect::<Vec<(DuplicateEntry, Vec<DuplicateEntry>)>>();
for (fe, vec_fe) in vec { for (fe, vec_fe) in vec {
self.files_with_identical_size_names_referenced self.files_with_identical_size_names_referenced
.insert((fe.size, fe.path.to_string_lossy().to_string()), (fe, vec_fe)); .insert((fe.size, fe.path.to_string_lossy().to_string()), (fe, vec_fe));
@ -306,19 +341,12 @@ impl DuplicateFinder {
_ => panic!(), _ => panic!(),
}; };
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .common_data(&self.common_data)
.group_by(|fe| fe.size) .group_by(|fe| fe.size)
.stop_receiver(stop_receiver) .stop_receiver(stop_receiver)
.progress_sender(progress_sender) .progress_sender(progress_sender)
.checking_method(self.check_method) .checking_method(self.check_method)
.max_stage(max_stage) .max_stage(max_stage)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.minimal_file_size(self.common_data.minimal_file_size)
.maximal_file_size(self.common_data.maximal_file_size)
.tool_type(self.common_data.tool_type)
.build() .build()
.run(); .run();
@ -334,7 +362,8 @@ impl DuplicateFinder {
let vector = if self.ignore_hard_links { filter_hard_links(&vec) } else { vec }; let vector = if self.ignore_hard_links { filter_hard_links(&vec) } else { vec };
if vector.len() > 1 { if vector.len() > 1 {
self.files_with_identical_size.insert(size, vector); self.files_with_identical_size
.insert(size, vector.into_iter().map(FileEntry::into_duplicate_entry).collect());
} }
} }
@ -388,7 +417,7 @@ impl DuplicateFinder {
Some((files_from_referenced_folders.pop().unwrap(), normal_files)) Some((files_from_referenced_folders.pop().unwrap(), normal_files))
} }
}) })
.collect::<Vec<(FileEntry, Vec<FileEntry>)>>(); .collect::<Vec<(DuplicateEntry, Vec<DuplicateEntry>)>>();
for (fe, vec_fe) in vec { for (fe, vec_fe) in vec {
self.files_with_identical_size_referenced.insert(fe.size, (fe, vec_fe)); self.files_with_identical_size_referenced.insert(fe.size, (fe, vec_fe));
} }
@ -396,17 +425,17 @@ impl DuplicateFinder {
} }
#[fun_time(message = "prehash_load_cache_at_start", level = "debug")] #[fun_time(message = "prehash_load_cache_at_start", level = "debug")]
fn prehash_load_cache_at_start(&mut self) -> (BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>) { fn prehash_load_cache_at_start(&mut self) -> (BTreeMap<u64, Vec<DuplicateEntry>>, BTreeMap<u64, Vec<DuplicateEntry>>, BTreeMap<u64, Vec<DuplicateEntry>>) {
// Cache algorithm // Cache algorithm
// - Load data from cache // - Load data from cache
// - Convert from BT<u64,Vec<FileEntry>> to BT<String,FileEntry> // - Convert from BT<u64,Vec<DuplicateEntry>> to BT<String,DuplicateEntry>
// - Save to proper values // - Save to proper values
let loaded_hash_map; let loaded_hash_map;
let mut records_already_cached: BTreeMap<u64, Vec<FileEntry>> = Default::default(); let mut records_already_cached: BTreeMap<u64, Vec<DuplicateEntry>> = Default::default();
let mut non_cached_files_to_check: BTreeMap<u64, Vec<FileEntry>> = Default::default(); let mut non_cached_files_to_check: BTreeMap<u64, Vec<DuplicateEntry>> = Default::default();
if self.use_prehash_cache { if self.use_prehash_cache {
let (messages, loaded_items) = load_cache_from_file_generalized_by_size::<FileEntry>( let (messages, loaded_items) = load_cache_from_file_generalized_by_size::<DuplicateEntry>(
&get_duplicate_cache_file(&self.hash_type, true), &get_duplicate_cache_file(&self.hash_type, true),
self.get_delete_outdated_cache(), self.get_delete_outdated_cache(),
&self.files_with_identical_size, &self.files_with_identical_size,
@ -418,7 +447,7 @@ impl DuplicateFinder {
for (size, mut vec_file_entry) in mem::take(&mut self.files_with_identical_size) { for (size, mut vec_file_entry) in mem::take(&mut self.files_with_identical_size) {
if let Some(cached_vec_file_entry) = loaded_hash_map.get(&size) { if let Some(cached_vec_file_entry) = loaded_hash_map.get(&size) {
// TODO maybe hashmap is not needed when using < 4 elements // TODO maybe hashmap is not needed when using < 4 elements
let mut cached_path_entries: HashMap<&Path, FileEntry> = HashMap::new(); let mut cached_path_entries: HashMap<&Path, DuplicateEntry> = HashMap::new();
for file_entry in cached_vec_file_entry { for file_entry in cached_vec_file_entry {
cached_path_entries.insert(&file_entry.path, file_entry.clone()); cached_path_entries.insert(&file_entry.path, file_entry.clone());
} }
@ -449,10 +478,14 @@ impl DuplicateFinder {
} }
#[fun_time(message = "prehash_save_cache_at_exit", level = "debug")] #[fun_time(message = "prehash_save_cache_at_exit", level = "debug")]
fn prehash_save_cache_at_exit(&mut self, loaded_hash_map: BTreeMap<u64, Vec<FileEntry>>, pre_hash_results: &Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)>) { fn prehash_save_cache_at_exit(
&mut self,
loaded_hash_map: BTreeMap<u64, Vec<DuplicateEntry>>,
pre_hash_results: &Vec<(u64, BTreeMap<String, Vec<DuplicateEntry>>, Vec<String>)>,
) {
if self.use_prehash_cache { if self.use_prehash_cache {
// All results = records already cached + computed results // All results = records already cached + computed results
let mut save_cache_to_hashmap: BTreeMap<String, FileEntry> = Default::default(); let mut save_cache_to_hashmap: BTreeMap<String, DuplicateEntry> = Default::default();
for (size, vec_file_entry) in loaded_hash_map { for (size, vec_file_entry) in loaded_hash_map {
if size >= self.minimal_prehash_cache_file_size { if size >= self.minimal_prehash_cache_file_size {
@ -487,7 +520,7 @@ impl DuplicateFinder {
&mut self, &mut self,
stop_receiver: Option<&Receiver<()>>, stop_receiver: Option<&Receiver<()>>,
progress_sender: Option<&Sender<ProgressData>>, progress_sender: Option<&Sender<ProgressData>>,
pre_checked_map: &mut BTreeMap<u64, Vec<FileEntry>>, pre_checked_map: &mut BTreeMap<u64, Vec<DuplicateEntry>>,
) -> Option<()> { ) -> Option<()> {
let check_type = self.hash_type; let check_type = self.hash_type;
let (progress_thread_handle, progress_thread_run, _atomic_counter, _check_was_stopped) = let (progress_thread_handle, progress_thread_run, _atomic_counter, _check_was_stopped) =
@ -510,10 +543,10 @@ impl DuplicateFinder {
debug!("Starting calculating prehash"); debug!("Starting calculating prehash");
#[allow(clippy::type_complexity)] #[allow(clippy::type_complexity)]
let pre_hash_results: Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)> = non_cached_files_to_check let pre_hash_results: Vec<(u64, BTreeMap<String, Vec<DuplicateEntry>>, Vec<String>)> = non_cached_files_to_check
.into_par_iter() .into_par_iter()
.map(|(size, vec_file_entry)| { .map(|(size, vec_file_entry)| {
let mut hashmap_with_hash: BTreeMap<String, Vec<FileEntry>> = Default::default(); let mut hashmap_with_hash: BTreeMap<String, Vec<DuplicateEntry>> = Default::default();
let mut errors: Vec<String> = Vec::new(); let mut errors: Vec<String> = Vec::new();
let mut buffer = [0u8; 1024 * 32]; let mut buffer = [0u8; 1024 * 32];
@ -526,7 +559,7 @@ impl DuplicateFinder {
match hash_calculation(&mut buffer, &file_entry, &check_type, 0) { match hash_calculation(&mut buffer, &file_entry, &check_type, 0) {
Ok(hash_string) => { Ok(hash_string) => {
file_entry.hash = hash_string.clone(); file_entry.hash = hash_string.clone();
hashmap_with_hash.entry(hash_string.clone()).or_default().push(file_entry); hashmap_with_hash.entry(hash_string).or_default().push(file_entry);
} }
Err(s) => errors.push(s), Err(s) => errors.push(s),
} }
@ -573,16 +606,16 @@ impl DuplicateFinder {
#[fun_time(message = "full_hashing_load_cache_at_start", level = "debug")] #[fun_time(message = "full_hashing_load_cache_at_start", level = "debug")]
fn full_hashing_load_cache_at_start( fn full_hashing_load_cache_at_start(
&mut self, &mut self,
mut pre_checked_map: BTreeMap<u64, Vec<FileEntry>>, mut pre_checked_map: BTreeMap<u64, Vec<DuplicateEntry>>,
) -> (BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>) { ) -> (BTreeMap<u64, Vec<DuplicateEntry>>, BTreeMap<u64, Vec<DuplicateEntry>>, BTreeMap<u64, Vec<DuplicateEntry>>) {
let loaded_hash_map; let loaded_hash_map;
let mut records_already_cached: BTreeMap<u64, Vec<FileEntry>> = Default::default(); let mut records_already_cached: BTreeMap<u64, Vec<DuplicateEntry>> = Default::default();
let mut non_cached_files_to_check: BTreeMap<u64, Vec<FileEntry>> = Default::default(); let mut non_cached_files_to_check: BTreeMap<u64, Vec<DuplicateEntry>> = Default::default();
if self.common_data.use_cache { if self.common_data.use_cache {
debug!("full_hashing_load_cache_at_start - using cache"); debug!("full_hashing_load_cache_at_start - using cache");
let (messages, loaded_items) = let (messages, loaded_items) =
load_cache_from_file_generalized_by_size::<FileEntry>(&get_duplicate_cache_file(&self.hash_type, false), self.get_delete_outdated_cache(), &pre_checked_map); load_cache_from_file_generalized_by_size::<DuplicateEntry>(&get_duplicate_cache_file(&self.hash_type, false), self.get_delete_outdated_cache(), &pre_checked_map);
self.get_text_messages_mut().extend_with_another_messages(messages); self.get_text_messages_mut().extend_with_another_messages(messages);
loaded_hash_map = loaded_items.unwrap_or_default(); loaded_hash_map = loaded_items.unwrap_or_default();
@ -590,7 +623,7 @@ impl DuplicateFinder {
for (size, mut vec_file_entry) in pre_checked_map { for (size, mut vec_file_entry) in pre_checked_map {
if let Some(cached_vec_file_entry) = loaded_hash_map.get(&size) { if let Some(cached_vec_file_entry) = loaded_hash_map.get(&size) {
// TODO maybe hashmap is not needed when using < 4 elements // TODO maybe hashmap is not needed when using < 4 elements
let mut cached_path_entries: HashMap<&Path, FileEntry> = HashMap::new(); let mut cached_path_entries: HashMap<&Path, DuplicateEntry> = HashMap::new();
for file_entry in cached_vec_file_entry { for file_entry in cached_vec_file_entry {
cached_path_entries.insert(&file_entry.path, file_entry.clone()); cached_path_entries.insert(&file_entry.path, file_entry.clone());
} }
@ -624,9 +657,9 @@ impl DuplicateFinder {
#[fun_time(message = "full_hashing_save_cache_at_exit", level = "debug")] #[fun_time(message = "full_hashing_save_cache_at_exit", level = "debug")]
fn full_hashing_save_cache_at_exit( fn full_hashing_save_cache_at_exit(
&mut self, &mut self,
records_already_cached: BTreeMap<u64, Vec<FileEntry>>, records_already_cached: BTreeMap<u64, Vec<DuplicateEntry>>,
full_hash_results: &mut Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)>, full_hash_results: &mut Vec<(u64, BTreeMap<String, Vec<DuplicateEntry>>, Vec<String>)>,
loaded_hash_map: BTreeMap<u64, Vec<FileEntry>>, loaded_hash_map: BTreeMap<u64, Vec<DuplicateEntry>>,
) { ) {
if !self.common_data.use_cache { if !self.common_data.use_cache {
return; return;
@ -642,7 +675,7 @@ impl DuplicateFinder {
} }
} }
// Size doesn't exists add results to files // Size doesn't exists add results to files
let mut temp_hashmap: BTreeMap<String, Vec<FileEntry>> = Default::default(); let mut temp_hashmap: BTreeMap<String, Vec<DuplicateEntry>> = Default::default();
for file_entry in vec_file_entry { for file_entry in vec_file_entry {
temp_hashmap.entry(file_entry.hash.clone()).or_default().push(file_entry); temp_hashmap.entry(file_entry.hash.clone()).or_default().push(file_entry);
} }
@ -650,7 +683,7 @@ impl DuplicateFinder {
} }
// Must save all results to file, old loaded from file with all currently counted results // Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = Default::default(); let mut all_results: BTreeMap<String, DuplicateEntry> = Default::default();
for (_size, vec_file_entry) in loaded_hash_map { for (_size, vec_file_entry) in loaded_hash_map {
for file_entry in vec_file_entry { for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry); all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
@ -674,7 +707,12 @@ impl DuplicateFinder {
} }
#[fun_time(message = "full_hashing", level = "debug")] #[fun_time(message = "full_hashing", level = "debug")]
fn full_hashing(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>, pre_checked_map: BTreeMap<u64, Vec<FileEntry>>) -> Option<()> { fn full_hashing(
&mut self,
stop_receiver: Option<&Receiver<()>>,
progress_sender: Option<&Sender<ProgressData>>,
pre_checked_map: BTreeMap<u64, Vec<DuplicateEntry>>,
) -> Option<()> {
let (progress_thread_handle, progress_thread_run, _atomic_counter, _check_was_stopped) = let (progress_thread_handle, progress_thread_run, _atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 4, MAX_STAGE, 0, self.check_method, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 4, MAX_STAGE, 0, self.check_method, self.common_data.tool_type);
@ -696,10 +734,10 @@ impl DuplicateFinder {
let check_type = self.hash_type; let check_type = self.hash_type;
debug!("Starting full hashing of {} files", non_cached_files_to_check.values().map(Vec::len).sum::<usize>()); debug!("Starting full hashing of {} files", non_cached_files_to_check.values().map(Vec::len).sum::<usize>());
let mut full_hash_results: Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)> = non_cached_files_to_check let mut full_hash_results: Vec<(u64, BTreeMap<String, Vec<DuplicateEntry>>, Vec<String>)> = non_cached_files_to_check
.into_par_iter() .into_par_iter()
.map(|(size, vec_file_entry)| { .map(|(size, vec_file_entry)| {
let mut hashmap_with_hash: BTreeMap<String, Vec<FileEntry>> = Default::default(); let mut hashmap_with_hash: BTreeMap<String, Vec<DuplicateEntry>> = Default::default();
let mut errors: Vec<String> = Vec::new(); let mut errors: Vec<String> = Vec::new();
let mut buffer = [0u8; 1024 * 16]; let mut buffer = [0u8; 1024 * 16];
@ -769,7 +807,7 @@ impl DuplicateFinder {
Some(all_results_with_same_size) Some(all_results_with_same_size)
} }
}) })
.collect::<Vec<Vec<(FileEntry, Vec<FileEntry>)>>>(); .collect::<Vec<Vec<(DuplicateEntry, Vec<DuplicateEntry>)>>>();
for vec_of_vec in vec { for vec_of_vec in vec {
self.files_with_identical_hashes_referenced.insert(vec_of_vec[0].0.size, vec_of_vec); self.files_with_identical_hashes_referenced.insert(vec_of_vec[0].0.size, vec_of_vec);
} }
@ -798,7 +836,7 @@ impl DuplicateFinder {
fn check_files_hash(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool { fn check_files_hash(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
assert_eq!(self.check_method, CheckingMethod::Hash); assert_eq!(self.check_method, CheckingMethod::Hash);
let mut pre_checked_map: BTreeMap<u64, Vec<FileEntry>> = Default::default(); let mut pre_checked_map: BTreeMap<u64, Vec<DuplicateEntry>> = Default::default();
let ret = self.prehashing(stop_receiver, progress_sender, &mut pre_checked_map); let ret = self.prehashing(stop_receiver, progress_sender, &mut pre_checked_map);
if ret.is_none() { if ret.is_none() {
return false; return false;
@ -834,7 +872,7 @@ impl DuplicateFinder {
} }
CheckingMethod::Hash => { CheckingMethod::Hash => {
for vec_files in self.files_with_identical_hashes.values() { for vec_files in self.files_with_identical_hashes.values() {
let vev: Vec<&Vec<FileEntry>> = vec_files.iter().collect::<Vec<_>>(); let vev: Vec<&Vec<DuplicateEntry>> = vec_files.iter().collect::<Vec<_>>();
delete_files_custom(&vev, &self.common_data.delete_method, &mut self.common_data.text_messages, self.common_data.dry_run); delete_files_custom(&vev, &self.common_data.delete_method, &mut self.common_data.text_messages, self.common_data.dry_run);
} }
} }
@ -864,7 +902,7 @@ impl DuplicateFinder {
self.minimal_prehash_cache_file_size = minimal_prehash_cache_file_size; self.minimal_prehash_cache_file_size = minimal_prehash_cache_file_size;
} }
pub const fn get_files_sorted_by_names(&self) -> &BTreeMap<String, Vec<FileEntry>> { pub const fn get_files_sorted_by_names(&self) -> &BTreeMap<String, Vec<DuplicateEntry>> {
&self.files_with_identical_names &self.files_with_identical_names
} }
@ -872,15 +910,15 @@ impl DuplicateFinder {
self.use_prehash_cache = use_prehash_cache; self.use_prehash_cache = use_prehash_cache;
} }
pub const fn get_files_sorted_by_size(&self) -> &BTreeMap<u64, Vec<FileEntry>> { pub const fn get_files_sorted_by_size(&self) -> &BTreeMap<u64, Vec<DuplicateEntry>> {
&self.files_with_identical_size &self.files_with_identical_size
} }
pub const fn get_files_sorted_by_size_name(&self) -> &BTreeMap<(u64, String), Vec<FileEntry>> { pub const fn get_files_sorted_by_size_name(&self) -> &BTreeMap<(u64, String), Vec<DuplicateEntry>> {
&self.files_with_identical_size_names &self.files_with_identical_size_names
} }
pub const fn get_files_sorted_by_hash(&self) -> &BTreeMap<u64, Vec<Vec<FileEntry>>> { pub const fn get_files_sorted_by_hash(&self) -> &BTreeMap<u64, Vec<Vec<DuplicateEntry>>> {
&self.files_with_identical_hashes &self.files_with_identical_hashes
} }
@ -908,19 +946,19 @@ impl DuplicateFinder {
self.common_data.use_reference_folders self.common_data.use_reference_folders
} }
pub fn get_files_with_identical_hashes_referenced(&self) -> &BTreeMap<u64, Vec<(FileEntry, Vec<FileEntry>)>> { pub fn get_files_with_identical_hashes_referenced(&self) -> &BTreeMap<u64, Vec<(DuplicateEntry, Vec<DuplicateEntry>)>> {
&self.files_with_identical_hashes_referenced &self.files_with_identical_hashes_referenced
} }
pub fn get_files_with_identical_name_referenced(&self) -> &BTreeMap<String, (FileEntry, Vec<FileEntry>)> { pub fn get_files_with_identical_name_referenced(&self) -> &BTreeMap<String, (DuplicateEntry, Vec<DuplicateEntry>)> {
&self.files_with_identical_names_referenced &self.files_with_identical_names_referenced
} }
pub fn get_files_with_identical_size_referenced(&self) -> &BTreeMap<u64, (FileEntry, Vec<FileEntry>)> { pub fn get_files_with_identical_size_referenced(&self) -> &BTreeMap<u64, (DuplicateEntry, Vec<DuplicateEntry>)> {
&self.files_with_identical_size_referenced &self.files_with_identical_size_referenced
} }
pub fn get_files_with_identical_size_names_referenced(&self) -> &BTreeMap<(u64, String), (FileEntry, Vec<FileEntry>)> { pub fn get_files_with_identical_size_names_referenced(&self) -> &BTreeMap<(u64, String), (DuplicateEntry, Vec<DuplicateEntry>)> {
&self.files_with_identical_size_names_referenced &self.files_with_identical_size_names_referenced
} }
} }
@ -1216,7 +1254,7 @@ pub trait MyHasher {
fn finalize(&self) -> String; fn finalize(&self) -> String;
} }
fn hash_calculation(buffer: &mut [u8], file_entry: &FileEntry, hash_type: &HashType, limit: u64) -> Result<String, String> { fn hash_calculation(buffer: &mut [u8], file_entry: &DuplicateEntry, hash_type: &HashType, limit: u64) -> Result<String, String> {
let mut file_handler = match File::open(&file_entry.path) { let mut file_handler = match File::open(&file_entry.path) {
Ok(t) => t, Ok(t) => t,
Err(e) => return Err(format!("Unable to check hash of file {:?}, reason {e}", file_entry.path)), Err(e) => return Err(format!("Unable to check hash of file {:?}, reason {e}", file_entry.path)),
@ -1376,7 +1414,7 @@ mod tests {
let src = dir.path().join("a"); let src = dir.path().join("a");
let mut file = File::create(&src)?; let mut file = File::create(&src)?;
file.write_all(b"aa")?; file.write_all(b"aa")?;
let e = FileEntry { path: src, ..Default::default() }; let e = DuplicateEntry { path: src, ..Default::default() };
let r = hash_calculation(&mut buf, &e, &HashType::Blake3, 0).unwrap(); let r = hash_calculation(&mut buf, &e, &HashType::Blake3, 0).unwrap();
assert!(!r.is_empty()); assert!(!r.is_empty());
Ok(()) Ok(())
@ -1389,7 +1427,7 @@ mod tests {
let src = dir.path().join("a"); let src = dir.path().join("a");
let mut file = File::create(&src)?; let mut file = File::create(&src)?;
file.write_all(b"aa")?; file.write_all(b"aa")?;
let e = FileEntry { path: src, ..Default::default() }; let e = DuplicateEntry { path: src, ..Default::default() };
let r1 = hash_calculation(&mut buf, &e, &HashType::Blake3, 1).unwrap(); let r1 = hash_calculation(&mut buf, &e, &HashType::Blake3, 1).unwrap();
let r2 = hash_calculation(&mut buf, &e, &HashType::Blake3, 2).unwrap(); let r2 = hash_calculation(&mut buf, &e, &HashType::Blake3, 2).unwrap();
let r3 = hash_calculation(&mut buf, &e, &HashType::Blake3, u64::MAX).unwrap(); let r3 = hash_calculation(&mut buf, &e, &HashType::Blake3, u64::MAX).unwrap();
@ -1403,7 +1441,7 @@ mod tests {
let dir = tempfile::Builder::new().tempdir()?; let dir = tempfile::Builder::new().tempdir()?;
let mut buf = [0u8; 1 << 10]; let mut buf = [0u8; 1 << 10];
let src = dir.path().join("a"); let src = dir.path().join("a");
let e = FileEntry { path: src, ..Default::default() }; let e = DuplicateEntry { path: src, ..Default::default() };
let r = hash_calculation(&mut buf, &e, &HashType::Blake3, 0).unwrap_err(); let r = hash_calculation(&mut buf, &e, &HashType::Blake3, 0).unwrap_err();
assert!(!r.is_empty()); assert!(!r.is_empty());
Ok(()) Ok(())

9
czkawka_core/src/empty_files.rs
View file

@ -53,17 +53,12 @@ impl EmptyFiles {
#[fun_time(message = "check_files", level = "debug")] #[fun_time(message = "check_files", level = "debug")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .common_data(&self.common_data)
.group_by(|_fe| ()) .group_by(|_fe| ())
.stop_receiver(stop_receiver) .stop_receiver(stop_receiver)
.progress_sender(progress_sender) .progress_sender(progress_sender)
.minimal_file_size(0) .minimal_file_size(0)
.maximal_file_size(0) .maximal_file_size(0)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.tool_type(self.common_data.tool_type)
.build() .build()
.run(); .run();
@ -87,7 +82,7 @@ impl EmptyFiles {
match self.common_data.delete_method { match self.common_data.delete_method {
DeleteMethod::Delete => { DeleteMethod::Delete => {
for file_entry in &self.empty_files { for file_entry in &self.empty_files {
if fs::remove_file(file_entry.path.clone()).is_err() { if fs::remove_file(&file_entry.path).is_err() {
self.common_data.text_messages.warnings.push(file_entry.path.to_string_lossy().to_string()); self.common_data.text_messages.warnings.push(file_entry.path.to_string_lossy().to_string());
} }
} }

2
czkawka_core/src/empty_folder.rs
View file

@ -310,7 +310,7 @@ impl PrintResults for EmptyFolder {
writeln!(writer, "--------------------------Empty folder list--------------------------")?; writeln!(writer, "--------------------------Empty folder list--------------------------")?;
writeln!(writer, "Found {} empty folders", self.information.number_of_empty_folders)?; writeln!(writer, "Found {} empty folders", self.information.number_of_empty_folders)?;
let mut empty_folder_list = self.empty_folder_list.keys().collect::<Vec<_>>(); let mut empty_folder_list = self.empty_folder_list.keys().collect::<Vec<_>>();
empty_folder_list.sort_unstable(); empty_folder_list.par_sort_unstable();
for name in empty_folder_list { for name in empty_folder_list {
writeln!(writer, "{name}")?; writeln!(writer, "{name}")?;
} }

108
czkawka_core/src/invalid_symlinks.rs
View file

@ -1,10 +1,12 @@
use std::fs; use std::fs;
use std::io::prelude::*; use std::io::prelude::*;
use std::path::{Path, PathBuf};
use crossbeam_channel::{Receiver, Sender}; use crossbeam_channel::{Receiver, Sender};
use fun_time::fun_time; use fun_time::fun_time;
use log::debug; use log::debug;
use serde::{Deserialize, Serialize};
use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, ErrorType, FileEntry, ProgressData, ToolType}; use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, ErrorType, FileEntry, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
@ -15,10 +17,50 @@ pub struct Info {
pub number_of_invalid_symlinks: usize, pub number_of_invalid_symlinks: usize,
} }
const MAX_NUMBER_OF_SYMLINK_JUMPS: i32 = 20;
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub struct SymlinkInfo {
pub destination_path: PathBuf,
pub type_of_error: ErrorType,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SymlinksFileEntry {
pub path: PathBuf,
pub size: u64,
pub modified_date: u64,
pub symlink_info: SymlinkInfo,
}
impl ResultEntry for SymlinksFileEntry {
fn get_path(&self) -> &Path {
&self.path
}
fn get_modified_date(&self) -> u64 {
self.modified_date
}
fn get_size(&self) -> u64 {
self.size
}
}
impl FileEntry {
fn into_symlinks_entry(self, symlink_info: SymlinkInfo) -> SymlinksFileEntry {
SymlinksFileEntry {
size: self.size,
path: self.path,
modified_date: self.modified_date,
symlink_info,
}
}
}
pub struct InvalidSymlinks { pub struct InvalidSymlinks {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
invalid_symlinks: Vec<FileEntry>, invalid_symlinks: Vec<SymlinksFileEntry>,
} }
impl InvalidSymlinks { impl InvalidSymlinks {
pub fn new() -> Self { pub fn new() -> Self {
@ -43,22 +85,26 @@ impl InvalidSymlinks {
#[fun_time(message = "check_files", level = "debug")] #[fun_time(message = "check_files", level = "debug")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone()) .common_data(&self.common_data)
.group_by(|_fe| ()) .group_by(|_fe| ())
.stop_receiver(stop_receiver) .stop_receiver(stop_receiver)
.progress_sender(progress_sender) .progress_sender(progress_sender)
.collect(Collect::InvalidSymlinks) .collect(Collect::InvalidSymlinks)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.tool_type(self.common_data.tool_type)
.build() .build()
.run(); .run();
match result { match result {
DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => { DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
self.invalid_symlinks = grouped_file_entries.into_values().flatten().collect(); self.invalid_symlinks = grouped_file_entries
.into_values()
.flatten()
.filter_map(|e| {
let Some((destination_path, type_of_error)) = Self::check_invalid_symlinks(&e.path) else {
return None;
};
Some(e.into_symlinks_entry(SymlinkInfo { destination_path, type_of_error }))
})
.collect();
self.information.number_of_invalid_symlinks = self.invalid_symlinks.len(); self.information.number_of_invalid_symlinks = self.invalid_symlinks.len();
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
debug!("Found {} invalid symlinks.", self.information.number_of_invalid_symlinks); debug!("Found {} invalid symlinks.", self.information.number_of_invalid_symlinks);
@ -68,12 +114,50 @@ impl InvalidSymlinks {
} }
} }
fn check_invalid_symlinks(current_file_name: &Path) -> Option<(PathBuf, ErrorType)> {
let mut destination_path = PathBuf::new();
let type_of_error;
match current_file_name.read_link() {
Ok(t) => {
destination_path.push(t);
let mut number_of_loop = 0;
let mut current_path = current_file_name.to_path_buf();
loop {
if number_of_loop == 0 && !current_path.exists() {
type_of_error = ErrorType::NonExistentFile;
break;
}
if number_of_loop == MAX_NUMBER_OF_SYMLINK_JUMPS {
type_of_error = ErrorType::InfiniteRecursion;
break;
}
current_path = match current_path.read_link() {
Ok(t) => t,
Err(_inspected) => {
// Looks that some next symlinks are broken, but we do nothing with it - TODO why they are broken
return None;
}
};
number_of_loop += 1;
}
}
Err(_inspected) => {
// Failed to load info about it
type_of_error = ErrorType::NonExistentFile;
}
}
Some((destination_path, type_of_error))
}
#[fun_time(message = "delete_files", level = "debug")] #[fun_time(message = "delete_files", level = "debug")]
fn delete_files(&mut self) { fn delete_files(&mut self) {
match self.common_data.delete_method { match self.common_data.delete_method {
DeleteMethod::Delete => { DeleteMethod::Delete => {
for file_entry in &self.invalid_symlinks { for file_entry in &self.invalid_symlinks {
if fs::remove_file(file_entry.path.clone()).is_err() { if fs::remove_file(&file_entry.path).is_err() {
self.common_data.text_messages.warnings.push(file_entry.path.to_string_lossy().to_string()); self.common_data.text_messages.warnings.push(file_entry.path.to_string_lossy().to_string());
} }
} }
@ -113,8 +197,8 @@ impl PrintResults for InvalidSymlinks {
writer, writer,
"{:?}\t\t{:?}\t\t{}", "{:?}\t\t{:?}\t\t{}",
file_entry.path, file_entry.path,
file_entry.symlink_info.clone().expect("invalid traversal result").destination_path, file_entry.symlink_info.destination_path,
match file_entry.symlink_info.clone().expect("invalid traversal result").type_of_error { match file_entry.symlink_info.type_of_error {
ErrorType::InfiniteRecursion => "Infinite Recursion", ErrorType::InfiniteRecursion => "Infinite Recursion",
ErrorType::NonExistentFile => "Non Existent File", ErrorType::NonExistentFile => "Non Existent File",
} }
@ -142,7 +226,7 @@ impl CommonData for InvalidSymlinks {
} }
impl InvalidSymlinks { impl InvalidSymlinks {
pub const fn get_invalid_symlinks(&self) -> &Vec<FileEntry> { pub const fn get_invalid_symlinks(&self) -> &Vec<SymlinksFileEntry> {
&self.invalid_symlinks &self.invalid_symlinks
} }

40
czkawka_core/src/same_music.rs
View file

@ -80,10 +80,10 @@ impl ResultEntry for MusicEntry {
} }
impl FileEntry { impl FileEntry {
fn to_music_entry(&self) -> MusicEntry { fn into_music_entry(self) -> MusicEntry {
MusicEntry { MusicEntry {
size: self.size, size: self.size,
path: self.path.clone(), path: self.path,
modified_date: self.modified_date, modified_date: self.modified_date,
fingerprint: vec![], fingerprint: vec![],
@ -177,13 +177,9 @@ impl SameMusic {
#[fun_time(message = "check_files", level = "debug")] #[fun_time(message = "check_files", level = "debug")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool { fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
if !self.common_data.allowed_extensions.using_custom_extensions() { self.common_data.allowed_extensions.set_and_validate_extensions(AUDIO_FILES_EXTENSIONS);
self.common_data.allowed_extensions.extend_allowed_extensions(AUDIO_FILES_EXTENSIONS); if !self.common_data.allowed_extensions.set_any_extensions() {
} else { return true;
self.common_data.allowed_extensions.extend_allowed_extensions(AUDIO_FILES_EXTENSIONS);
if !self.common_data.allowed_extensions.using_custom_extensions() {
return true;
}
} }
let max_stage = match self.check_type { let max_stage = match self.check_type {
@ -193,17 +189,10 @@ impl SameMusic {
}; };
let result = DirTraversalBuilder::new() let result = DirTraversalBuilder::new()
.root_dirs(self.common_data.directories.included_directories.clone())
.group_by(|_fe| ()) .group_by(|_fe| ())
.stop_receiver(stop_receiver) .stop_receiver(stop_receiver)
.progress_sender(progress_sender) .progress_sender(progress_sender)
.minimal_file_size(self.common_data.minimal_file_size) .common_data(&self.common_data)
.maximal_file_size(self.common_data.maximal_file_size)
.directories(self.common_data.directories.clone())
.allowed_extensions(self.common_data.allowed_extensions.clone())
.excluded_items(self.common_data.excluded_items.clone())
.recursive_search(self.common_data.recursive_search)
.tool_type(self.common_data.tool_type)
.max_stage(max_stage) .max_stage(max_stage)
.build() .build()
.run(); .run();
@ -213,7 +202,7 @@ impl SameMusic {
self.music_to_check = grouped_file_entries self.music_to_check = grouped_file_entries
.into_values() .into_values()
.flatten() .flatten()
.map(|fe| (fe.path.to_string_lossy().to_string(), fe.to_music_entry())) .map(|fe| (fe.path.to_string_lossy().to_string(), fe.into_music_entry()))
.collect(); .collect();
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
debug!("check_files - Found {} music files.", self.music_to_check.len()); debug!("check_files - Found {} music files.", self.music_to_check.len());
@ -834,7 +823,7 @@ fn read_single_file_tag(path: &str, music_entry: &mut MusicEntry) -> bool {
return false; return false;
}; };
let result = panic::catch_unwind(move || { let Ok(possible_tagged_file) = panic::catch_unwind(move || {
match read_from(&mut file) { match read_from(&mut file) {
Ok(t) => Some(t), Ok(t) => Some(t),
Err(_inspected) => { Err(_inspected) => {
@ -842,21 +831,14 @@ fn read_single_file_tag(path: &str, music_entry: &mut MusicEntry) -> bool {
None None
} }
} }
}); }) else {
let tagged_file = if let Ok(t) = result {
match t {
Some(r) => r,
None => {
return true;
}
}
} else {
let message = create_crash_message("Lofty", path, "https://github.com/image-rs/image/issues"); let message = create_crash_message("Lofty", path, "https://github.com/image-rs/image/issues");
println!("{message}"); println!("{message}");
return false; return false;
}; };
let Some(tagged_file) = possible_tagged_file else { return true };
let properties = tagged_file.properties(); let properties = tagged_file.properties();
let mut track_title = String::new(); let mut track_title = String::new();

355
czkawka_core/src/similar_images.rs
View file

@ -1,5 +1,4 @@
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::fs::DirEntry;
use std::io::Write; use std::io::Write;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
@ -20,11 +19,11 @@ use serde::{Deserialize, Serialize};
#[cfg(feature = "heif")] #[cfg(feature = "heif")]
use crate::common::get_dynamic_image_from_heic; use crate::common::get_dynamic_image_from_heic;
use crate::common::{ use crate::common::{
check_folder_children, check_if_stop_received, create_crash_message, delete_files_custom, get_dynamic_image_from_raw_image, prepare_thread_handler_common, check_if_stop_received, create_crash_message, delete_files_custom, get_dynamic_image_from_raw_image, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads,
send_info_and_wait_for_ending_all_threads, HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS, HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS,
}; };
use crate::common_cache::{get_similar_images_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_cache::{get_similar_images_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
use crate::common_dir_traversal::{common_read_dir, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry}; use crate::common_traits::{DebugPrint, PrintResults, ResultEntry};
use crate::flc; use crate::flc;
@ -39,16 +38,17 @@ pub const SIMILAR_VALUES: [[u32; 6]; 4] = [
]; ];
#[derive(Clone, Debug, Serialize, Deserialize)] #[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileEntry { pub struct ImagesEntry {
pub path: PathBuf, pub path: PathBuf,
pub size: u64, pub size: u64,
pub dimensions: String, pub dimensions: String,
pub modified_date: u64, pub modified_date: u64,
pub hash: ImHash, pub hash: ImHash,
pub similarity: u32, pub similarity: u32,
pub image_type: ImageType,
} }
impl ResultEntry for FileEntry { impl ResultEntry for ImagesEntry {
fn get_path(&self) -> &Path { fn get_path(&self) -> &Path {
&self.path &self.path
} }
@ -59,6 +59,30 @@ impl ResultEntry for FileEntry {
self.size self.size
} }
} }
impl FileEntry {
fn into_images_entry(self) -> ImagesEntry {
ImagesEntry {
size: self.size,
path: self.path,
modified_date: self.modified_date,
dimensions: String::new(),
hash: Vec::new(),
similarity: 0,
image_type: ImageType::Unknown,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub enum ImageType {
Normal,
Raw,
Heic,
Unknown,
}
const MAX_IMAGE_STAGE: u8 = 2;
#[derive(Clone, Debug, Copy)] #[derive(Clone, Debug, Copy)]
pub enum SimilarityPreset { pub enum SimilarityPreset {
@ -88,13 +112,13 @@ pub struct SimilarImages {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
bktree: BKTree<ImHash, Hamming>, bktree: BKTree<ImHash, Hamming>,
similar_vectors: Vec<Vec<FileEntry>>, similar_vectors: Vec<Vec<ImagesEntry>>,
similar_referenced_vectors: Vec<(FileEntry, Vec<FileEntry>)>, similar_referenced_vectors: Vec<(ImagesEntry, Vec<ImagesEntry>)>,
image_hashes: HashMap<ImHash, Vec<FileEntry>>,
// Hashmap with image hashes and Vector with names of files // Hashmap with image hashes and Vector with names of files
image_hashes: HashMap<ImHash, Vec<ImagesEntry>>,
similarity: u32, similarity: u32,
images_to_check: BTreeMap<String, FileEntry>, images_to_check: BTreeMap<String, ImagesEntry>,
pub hash_size: u8, // TODO to remove pub, this is needeed by new gui, because there is no way to check what exactly was seelected hash_size: u8,
hash_alg: HashAlg, hash_alg: HashAlg,
image_filter: FilterType, image_filter: FilterType,
exclude_images_with_same_size: bool, exclude_images_with_same_size: bool,
@ -144,133 +168,71 @@ impl SimilarImages {
self.debug_print(); self.debug_print();
} }
#[fun_time(message = "check_for_similar_images", level = "debug")] // #[fun_time(message = "check_for_similar_images", level = "debug")]
fn check_for_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool { fn check_for_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
let mut folders_to_check: Vec<PathBuf> = self.common_data.directories.included_directories.clone(); if cfg!(feature = "heif") {
self.common_data
if !self.common_data.allowed_extensions.using_custom_extensions() { .allowed_extensions
self.common_data.allowed_extensions.extend_allowed_extensions(IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS); .set_and_validate_extensions(&[IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS, HEIC_EXTENSIONS].concat());
self.common_data.allowed_extensions.extend_allowed_extensions(RAW_IMAGE_EXTENSIONS);
#[cfg(feature = "heif")]
self.common_data.allowed_extensions.extend_allowed_extensions(HEIC_EXTENSIONS);
} else { } else {
self.common_data self.common_data
.allowed_extensions .allowed_extensions
.extend_allowed_extensions(&[IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS, HEIC_EXTENSIONS].concat()); .set_and_validate_extensions(&[IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS].concat());
if !self.common_data.allowed_extensions.using_custom_extensions() { }
return true; if !self.common_data.allowed_extensions.set_any_extensions() {
} return true;
} }
// Add root folders for finding let normal_image_extensions = IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS.iter().collect::<HashSet<_>>();
for id in &self.common_data.directories.included_directories { let raw_image_extensions = RAW_IMAGE_EXTENSIONS.iter().collect::<HashSet<_>>();
folders_to_check.push(id.clone()); let heic_extensions = HEIC_EXTENSIONS.iter().collect::<HashSet<_>>();
}
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = let result = DirTraversalBuilder::new()
prepare_thread_handler_common(progress_sender, 0, 2, 0, CheckingMethod::None, self.common_data.tool_type); .group_by(|_fe| ())
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.common_data(&self.common_data)
.max_stage(MAX_IMAGE_STAGE)
.build()
.run();
while !folders_to_check.is_empty() { match result {
if check_if_stop_received(stop_receiver) { DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); self.images_to_check = grouped_file_entries
return false; .into_values()
} .flatten()
.map(|fe| {
let segments: Vec<_> = folders_to_check let fe_str = fe.path.to_string_lossy().to_string();
.into_par_iter() let extension_lowercase = fe.path.extension().unwrap_or_default().to_string_lossy().to_lowercase();
.map(|current_folder| { let mut image_entry = fe.into_images_entry();
let mut dir_result = vec![]; if normal_image_extensions.contains(&extension_lowercase.as_str()) {
let mut warnings = vec![]; image_entry.image_type = ImageType::Normal;
let mut fe_result = vec![]; } else if raw_image_extensions.contains(&extension_lowercase.as_str()) {
image_entry.image_type = ImageType::Raw;
let Some(read_dir) = common_read_dir(&current_folder, &mut warnings) else { } else if heic_extensions.contains(&extension_lowercase.as_str()) {
return (dir_result, warnings, fe_result); image_entry.image_type = ImageType::Heic;
};
for entry in read_dir {
let Ok(entry_data) = entry else {
continue;
};
let Ok(file_type) = entry_data.file_type() else {
continue;
};
if file_type.is_dir() {
check_folder_children(
&mut dir_result,
&mut warnings,
&entry_data,
self.common_data.recursive_search,
&self.common_data.directories,
&self.common_data.excluded_items,
);
} else if file_type.is_file() {
atomic_counter.fetch_add(1, Ordering::Relaxed);
self.add_file_entry(&entry_data, &mut fe_result, &mut warnings);
} }
} (fe_str, image_entry)
(dir_result, warnings, fe_result) })
}) .collect();
.collect();
let required_size = segments.iter().map(|(segment, _, _)| segment.len()).sum::<usize>();
folders_to_check = Vec::with_capacity(required_size);
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
for (name, fe) in fe_result { debug!("check_files - Found {} image files.", self.images_to_check.len());
self.images_to_check.insert(name, fe); true
}
} }
}
eprintln!("Tested {} files", atomic_counter.load(Ordering::Relaxed));
eprintln!("Imagest to check {}", self.images_to_check.len());
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); DirTraversalResult::Stopped => false,
true
}
fn add_file_entry(&self, entry_data: &DirEntry, fe_result: &mut Vec<(String, FileEntry)>, warnings: &mut Vec<String>) {
if !self.common_data.allowed_extensions.check_if_entry_ends_with_extension(entry_data) {
return;
}
let current_file_name = entry_data.path();
if self.common_data.excluded_items.is_excluded(&current_file_name) {
return;
}
let Ok(metadata) = entry_data.metadata() else {
return;
};
// Checking files
if (self.common_data.minimal_file_size..=self.common_data.maximal_file_size).contains(&metadata.len()) {
let path_str = current_file_name.to_string_lossy().to_string();
let fe: FileEntry = FileEntry {
size: metadata.len(),
dimensions: String::new(),
modified_date: get_modified_time(&metadata, warnings, &current_file_name, false),
path: current_file_name,
hash: Vec::new(),
similarity: 0,
};
fe_result.push((path_str, fe));
} }
} }
#[fun_time(message = "hash_images_load_cache", level = "debug")] #[fun_time(message = "hash_images_load_cache", level = "debug")]
fn hash_images_load_cache(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) { fn hash_images_load_cache(&mut self) -> (BTreeMap<String, ImagesEntry>, BTreeMap<String, ImagesEntry>, BTreeMap<String, ImagesEntry>) {
let loaded_hash_map; let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default(); let mut records_already_cached: BTreeMap<String, ImagesEntry> = Default::default();
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default(); let mut non_cached_files_to_check: BTreeMap<String, ImagesEntry> = Default::default();
if self.common_data.use_cache { if self.common_data.use_cache {
let (messages, loaded_items) = load_cache_from_file_generalized_by_path::<FileEntry>( let (messages, loaded_items) = load_cache_from_file_generalized_by_path::<ImagesEntry>(
&get_similar_images_cache_file(&self.hash_size, &self.hash_alg, &self.image_filter), &get_similar_images_cache_file(&self.hash_size, &self.hash_alg, &self.image_filter),
self.get_delete_outdated_cache(), self.get_delete_outdated_cache(),
&self.images_to_check, &self.images_to_check,
@ -315,34 +277,35 @@ impl SimilarImages {
prepare_thread_handler_common(progress_sender, 1, 2, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 1, 2, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type);
debug!("hash_images - start hashing images"); debug!("hash_images - start hashing images");
let mut vec_file_entry: Vec<(FileEntry, ImHash)> = non_cached_files_to_check let mut vec_file_entry: Vec<ImagesEntry> = non_cached_files_to_check
.into_par_iter() .into_par_iter()
.map(|(_s, file_entry)| { .map(|(_s, mut file_entry)| {
atomic_counter.fetch_add(1, Ordering::Relaxed); atomic_counter.fetch_add(1, Ordering::Relaxed);
if check_if_stop_received(stop_receiver) { if check_if_stop_received(stop_receiver) {
check_was_stopped.store(true, Ordering::Relaxed); check_was_stopped.store(true, Ordering::Relaxed);
return None; return None;
} }
Some(Some(self.collect_image_file_entry(file_entry))) self.collect_image_file_entry(&mut file_entry);
Some(Some(file_entry))
}) })
.while_some() .while_some()
.filter(Option::is_some) .filter_map(|e| e)
.map(Option::unwrap) .collect::<Vec<ImagesEntry>>();
.collect::<Vec<(FileEntry, ImHash)>>();
debug!("hash_images - end hashing images"); debug!("hash_images - end hashing images");
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Just connect loaded results with already calculated hashes // Just connect loaded results with already calculated hashes
for file_entry in records_already_cached.into_values() { for file_entry in records_already_cached.into_values() {
vec_file_entry.push((file_entry.clone(), file_entry.hash)); vec_file_entry.push(file_entry);
} }
// All valid entries are used to create bktree used to check for hash similarity // All valid entries are used to create bktree used to check for hash similarity
for (file_entry, buf) in &vec_file_entry { for file_entry in &vec_file_entry {
// Only use to comparing, non broken hashes(all 0 or 255 hashes means that algorithm fails to decode them because e.g. contains a log of alpha channel) // Only use to comparing, non broken hashes(all 0 or 255 hashes means that algorithm fails to decode them because e.g. contains a lot of alpha channel)
if !(buf.is_empty() || buf.iter().all(|e| *e == 0) || buf.iter().all(|e| *e == 255)) { if !(file_entry.hash.is_empty() || file_entry.hash.iter().all(|e| *e == 0) || file_entry.hash.iter().all(|e| *e == 255)) {
self.image_hashes.entry(buf.clone()).or_default().push(file_entry.clone()); self.image_hashes.entry(file_entry.hash.clone()).or_default().push(file_entry.clone());
} }
} }
@ -357,11 +320,11 @@ impl SimilarImages {
} }
#[fun_time(message = "save_to_cache", level = "debug")] #[fun_time(message = "save_to_cache", level = "debug")]
fn save_to_cache(&mut self, vec_file_entry: Vec<(FileEntry, ImHash)>, loaded_hash_map: BTreeMap<String, FileEntry>) { fn save_to_cache(&mut self, vec_file_entry: Vec<ImagesEntry>, loaded_hash_map: BTreeMap<String, ImagesEntry>) {
if self.common_data.use_cache { if self.common_data.use_cache {
// Must save all results to file, old loaded from file with all currently counted results // Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = loaded_hash_map; let mut all_results: BTreeMap<String, ImagesEntry> = loaded_hash_map;
for (file_entry, _hash) in vec_file_entry { for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry); all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
} }
@ -375,60 +338,61 @@ impl SimilarImages {
} }
} }
fn collect_image_file_entry(&self, mut file_entry: FileEntry) -> (FileEntry, ImHash) { fn collect_image_file_entry(&self, file_entry: &mut ImagesEntry) {
let file_name_lowercase = file_entry.path.to_string_lossy().to_lowercase(); let img;
let image;
#[allow(clippy::never_loop)] // Required to implement nice if/else
'krztyna: loop {
if RAW_IMAGE_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) {
image = match get_dynamic_image_from_raw_image(&file_entry.path) {
Some(t) => t,
None => return (file_entry, Vec::new()),
};
break 'krztyna;
}
if file_entry.image_type == ImageType::Heic {
#[cfg(feature = "heif")] #[cfg(feature = "heif")]
if HEIC_EXTENSIONS.iter().any(|e| file_name_lowercase.ends_with(e)) { {
image = match get_dynamic_image_from_heic(&file_entry.path.to_string_lossy()) { img = match get_dynamic_image_from_heic(&file_entry.path.to_string_lossy()) {
Ok(t) => t, Ok(t) => t,
Err(_) => { Err(_) => {
return (file_entry, Vec::new()); return;
} }
}; };
break 'krztyna;
} }
#[cfg(not(feature = "heif"))]
// Normal image extension, when any other fail, not using if/else {
let result = panic::catch_unwind(|| { if let Ok(image_result) = panic::catch_unwind(|| image::open(&file_entry.path)) {
match image::open(file_entry.path.clone()) { if let Ok(image2) = image_result {
Ok(t) => Ok(t), img = image2;
// Err(_inspected) => return Some(None), // Something is wrong with image, } else {
// For broken images empty hash is used, because without it will try to resecan files each time when it is called(missing cache file is responsible for it) return;
// This may cause problems(very rarely), when e.g. file was not available due lack of permissions, but it is available now }
Err(_inspected) => Err(()),
}
});
// If image crashed during opening, we just skip checking its hash and go on
if let Ok(image_result) = result {
if let Ok(image2) = image_result {
image = image2;
} else { } else {
return (file_entry, Vec::new()); let message = crate::common::create_crash_message("Image-rs", &file_entry.path.to_string_lossy(), "https://github.com/image-rs/image/issues");
println!("{message}");
return;
}
}
} else {
match file_entry.image_type {
ImageType::Normal | ImageType::Heic => {
if let Ok(image_result) = panic::catch_unwind(|| image::open(&file_entry.path)) {
if let Ok(image2) = image_result {
img = image2;
} else {
return;
}
} else {
let message = create_crash_message("Image-rs", &file_entry.path.to_string_lossy(), "https://github.com/image-rs/image/issues");
println!("{message}");
return;
}
}
ImageType::Raw => {
img = match get_dynamic_image_from_raw_image(&file_entry.path) {
Some(t) => t,
None => return,
};
}
_ => {
unreachable!();
} }
} else {
let message = create_crash_message("Image-rs", &file_entry.path.to_string_lossy(), "https://github.com/image-rs/image/issues");
println!("{message}");
return (file_entry, Vec::new());
} }
break 'krztyna;
} }
let dimensions = image.dimensions(); let dimensions = img.dimensions();
file_entry.dimensions = format!("{}x{}", dimensions.0, dimensions.1); file_entry.dimensions = format!("{}x{}", dimensions.0, dimensions.1);
@ -438,17 +402,13 @@ impl SimilarImages {
.resize_filter(self.image_filter); .resize_filter(self.image_filter);
let hasher = hasher_config.to_hasher(); let hasher = hasher_config.to_hasher();
let hash = hasher.hash_image(&image); let hash = hasher.hash_image(&img);
let buf: ImHash = hash.as_bytes().to_vec(); file_entry.hash = hash.as_bytes().to_vec();
file_entry.hash = buf.clone();
(file_entry, buf)
} }
// Split hashes at 2 parts, base hashes and hashes to compare, 3 argument is set of hashes with multiple images // Split hashes at 2 parts, base hashes and hashes to compare, 3 argument is set of hashes with multiple images
#[fun_time(message = "split_hashes", level = "debug")] #[fun_time(message = "split_hashes", level = "debug")]
fn split_hashes(&mut self, all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>) -> (Vec<ImHash>, HashSet<ImHash>) { fn split_hashes(&mut self, all_hashed_images: &HashMap<ImHash, Vec<ImagesEntry>>) -> (Vec<ImHash>, HashSet<ImHash>) {
let hashes_with_multiple_images: HashSet<ImHash> = all_hashed_images let hashes_with_multiple_images: HashSet<ImHash> = all_hashed_images
.iter() .iter()
.filter_map(|(hash, vec_file_entry)| { .filter_map(|(hash, vec_file_entry)| {
@ -460,8 +420,8 @@ impl SimilarImages {
.collect(); .collect();
let mut base_hashes = Vec::new(); // Initial hashes let mut base_hashes = Vec::new(); // Initial hashes
if self.common_data.use_reference_folders { if self.common_data.use_reference_folders {
let mut files_from_referenced_folders: HashMap<ImHash, Vec<FileEntry>> = HashMap::new(); let mut files_from_referenced_folders: HashMap<ImHash, Vec<ImagesEntry>> = HashMap::new();
let mut normal_files: HashMap<ImHash, Vec<FileEntry>> = HashMap::new(); let mut normal_files: HashMap<ImHash, Vec<ImagesEntry>> = HashMap::new();
all_hashed_images.clone().into_iter().for_each(|(hash, vec_file_entry)| { all_hashed_images.clone().into_iter().for_each(|(hash, vec_file_entry)| {
for file_entry in vec_file_entry { for file_entry in vec_file_entry {
@ -494,8 +454,8 @@ impl SimilarImages {
&self, &self,
hashes_parents: HashMap<ImHash, u32>, hashes_parents: HashMap<ImHash, u32>,
hashes_with_multiple_images: &HashSet<ImHash>, hashes_with_multiple_images: &HashSet<ImHash>,
all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>, all_hashed_images: &HashMap<ImHash, Vec<ImagesEntry>>,
collected_similar_images: &mut HashMap<ImHash, Vec<FileEntry>>, collected_similar_images: &mut HashMap<ImHash, Vec<ImagesEntry>>,
hashes_similarity: HashMap<ImHash, (ImHash, u32)>, hashes_similarity: HashMap<ImHash, (ImHash, u32)>,
) { ) {
if self.common_data.use_reference_folders { if self.common_data.use_reference_folders {
@ -552,8 +512,8 @@ impl SimilarImages {
#[fun_time(message = "compare_hashes_with_non_zero_tolerance", level = "debug")] #[fun_time(message = "compare_hashes_with_non_zero_tolerance", level = "debug")]
fn compare_hashes_with_non_zero_tolerance( fn compare_hashes_with_non_zero_tolerance(
&mut self, &mut self,
all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>, all_hashed_images: &HashMap<ImHash, Vec<ImagesEntry>>,
collected_similar_images: &mut HashMap<ImHash, Vec<FileEntry>>, collected_similar_images: &mut HashMap<ImHash, Vec<ImagesEntry>>,
progress_sender: Option<&Sender<ProgressData>>, progress_sender: Option<&Sender<ProgressData>>,
stop_receiver: Option<&Receiver<()>>, stop_receiver: Option<&Receiver<()>>,
tolerance: u32, tolerance: u32,
@ -697,7 +657,7 @@ impl SimilarImages {
let tolerance = self.similarity; let tolerance = self.similarity;
// Results // Results
let mut collected_similar_images: HashMap<ImHash, Vec<FileEntry>> = Default::default(); let mut collected_similar_images: HashMap<ImHash, Vec<ImagesEntry>> = Default::default();
let all_hashed_images = mem::take(&mut self.image_hashes); let all_hashed_images = mem::take(&mut self.image_hashes);
@ -776,13 +736,13 @@ impl SimilarImages {
Some((files_from_referenced_folders.pop().unwrap(), normal_files)) Some((files_from_referenced_folders.pop().unwrap(), normal_files))
} }
}) })
.collect::<Vec<(FileEntry, Vec<FileEntry>)>>(); .collect::<Vec<(ImagesEntry, Vec<ImagesEntry>)>>();
} }
} }
#[allow(unused_variables)] #[allow(unused_variables)]
// TODO this probably not works good when reference folders are used // TODO this probably not works good when reference folders are used
pub fn verify_duplicated_items(&self, collected_similar_images: &HashMap<ImHash, Vec<FileEntry>>) { pub fn verify_duplicated_items(&self, collected_similar_images: &HashMap<ImHash, Vec<ImagesEntry>>) {
if !cfg!(debug_assertions) { if !cfg!(debug_assertions) {
return; return;
} }
@ -1028,7 +988,7 @@ fn debug_check_for_duplicated_things(
use_reference_folders: bool, use_reference_folders: bool,
hashes_parents: &HashMap<ImHash, u32>, hashes_parents: &HashMap<ImHash, u32>,
hashes_similarity: &HashMap<ImHash, (ImHash, u32)>, hashes_similarity: &HashMap<ImHash, (ImHash, u32)>,
all_hashed_images: &HashMap<ImHash, Vec<FileEntry>>, all_hashed_images: &HashMap<ImHash, Vec<ImagesEntry>>,
numm: &str, numm: &str,
) { ) {
if !cfg!(debug_assertions) { if !cfg!(debug_assertions) {
@ -1111,11 +1071,11 @@ impl SimilarImages {
self.image_filter = image_filter; self.image_filter = image_filter;
} }
pub const fn get_similar_images(&self) -> &Vec<Vec<FileEntry>> { pub const fn get_similar_images(&self) -> &Vec<Vec<ImagesEntry>> {
&self.similar_vectors &self.similar_vectors
} }
pub fn get_similar_images_referenced(&self) -> &Vec<(FileEntry, Vec<FileEntry>)> { pub fn get_similar_images_referenced(&self) -> &Vec<(ImagesEntry, Vec<ImagesEntry>)> {
&self.similar_referenced_vectors &self.similar_referenced_vectors
} }
@ -1142,7 +1102,7 @@ mod tests {
use crate::common_directory::Directories; use crate::common_directory::Directories;
use crate::common_tool::CommonToolData; use crate::common_tool::CommonToolData;
use crate::similar_images::{FileEntry, Hamming, ImHash, SimilarImages}; use crate::similar_images::{Hamming, ImHash, ImageType, ImagesEntry, SimilarImages};
#[test] #[test]
fn test_compare_no_images() { fn test_compare_no_images() {
@ -1568,20 +1528,21 @@ mod tests {
assert_eq!(similarity, 1); assert_eq!(similarity, 1);
} }
fn add_hashes(hashmap: &mut HashMap<ImHash, Vec<FileEntry>>, file_entries: Vec<FileEntry>) { fn add_hashes(hashmap: &mut HashMap<ImHash, Vec<ImagesEntry>>, file_entries: Vec<ImagesEntry>) {
for fe in file_entries { for fe in file_entries {
hashmap.entry(fe.hash.clone()).or_default().push(fe); hashmap.entry(fe.hash.clone()).or_default().push(fe);
} }
} }
fn create_random_file_entry(hash: Vec<u8>, name: &str) -> FileEntry { fn create_random_file_entry(hash: Vec<u8>, name: &str) -> ImagesEntry {
FileEntry { ImagesEntry {
path: PathBuf::from(name.to_string()), path: PathBuf::from(name.to_string()),
size: 0, size: 0,
dimensions: String::new(), dimensions: String::new(),
modified_date: 0, modified_date: 0,
hash, hash,
similarity: 0, similarity: 0,
image_type: ImageType::Normal,
} }
} }
} }

184
czkawka_core/src/similar_videos.rs
View file

@ -1,5 +1,4 @@
use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fs::DirEntry;
use std::io::Write; use std::io::Write;
use std::mem; use std::mem;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
@ -9,16 +8,15 @@ use crossbeam_channel::{Receiver, Sender};
use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound; use ffmpeg_cmdline_utils::FfmpegErrorKind::FfmpegNotFound;
use fun_time::fun_time; use fun_time::fun_time;
use humansize::{format_size, BINARY}; use humansize::{format_size, BINARY};
use log::debug;
use rayon::prelude::*; use rayon::prelude::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo; use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
use vid_dup_finder_lib::{NormalizedTolerance, VideoHash}; use vid_dup_finder_lib::{NormalizedTolerance, VideoHash};
use crate::common::{ use crate::common::{check_if_stop_received, delete_files_custom, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS};
check_folder_children, check_if_stop_received, delete_files_custom, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS,
};
use crate::common_cache::{get_similar_videos_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_cache::{get_similar_videos_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
use crate::common_dir_traversal::{common_read_dir, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry}; use crate::common_traits::{DebugPrint, PrintResults, ResultEntry};
use crate::flc; use crate::flc;
@ -27,7 +25,7 @@ use crate::localizer_core::generate_translation_hashmap;
pub const MAX_TOLERANCE: i32 = 20; pub const MAX_TOLERANCE: i32 = 20;
#[derive(Clone, Debug, Serialize, Deserialize)] #[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FileEntry { pub struct VideosEntry {
pub path: PathBuf, pub path: PathBuf,
pub size: u64, pub size: u64,
pub modified_date: u64, pub modified_date: u64,
@ -35,7 +33,7 @@ pub struct FileEntry {
pub error: String, pub error: String,
} }
impl ResultEntry for FileEntry { impl ResultEntry for VideosEntry {
fn get_path(&self) -> &Path { fn get_path(&self) -> &Path {
&self.path &self.path
} }
@ -47,6 +45,19 @@ impl ResultEntry for FileEntry {
} }
} }
impl FileEntry {
fn into_videos_entry(self) -> VideosEntry {
VideosEntry {
size: self.size,
path: self.path,
modified_date: self.modified_date,
vhash: Default::default(),
error: String::new(),
}
}
}
struct Hamming; struct Hamming;
impl bk_tree::Metric<Vec<u8>> for Hamming { impl bk_tree::Metric<Vec<u8>> for Hamming {
@ -61,13 +72,15 @@ impl bk_tree::Metric<Vec<u8>> for Hamming {
} }
} }
const MAX_VIDEOS_STAGE: u8 = 1;
pub struct SimilarVideos { pub struct SimilarVideos {
common_data: CommonToolData, common_data: CommonToolData,
information: Info, information: Info,
similar_vectors: Vec<Vec<FileEntry>>, similar_vectors: Vec<Vec<VideosEntry>>,
similar_referenced_vectors: Vec<(FileEntry, Vec<FileEntry>)>, similar_referenced_vectors: Vec<(VideosEntry, Vec<VideosEntry>)>,
videos_hashes: BTreeMap<Vec<u8>, Vec<FileEntry>>, videos_hashes: BTreeMap<Vec<u8>, Vec<VideosEntry>>,
videos_to_check: BTreeMap<String, FileEntry>, videos_to_check: BTreeMap<String, VideosEntry>,
tolerance: i32, tolerance: i32,
exclude_videos_with_same_size: bool, exclude_videos_with_same_size: bool,
} }
@ -128,122 +141,47 @@ impl SimilarVideos {
self.debug_print(); self.debug_print();
} }
#[fun_time(message = "check_for_similar_videos", level = "debug")] // #[fun_time(message = "check_for_similar_videos", level = "debug")]
fn check_for_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool { fn check_for_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
let mut folders_to_check: Vec<PathBuf> = self.common_data.directories.included_directories.clone(); self.common_data.allowed_extensions.set_and_validate_extensions(VIDEO_FILES_EXTENSIONS);
if !self.common_data.allowed_extensions.set_any_extensions() {
if !self.common_data.allowed_extensions.using_custom_extensions() { return true;
self.common_data.allowed_extensions.extend_allowed_extensions(VIDEO_FILES_EXTENSIONS);
} else {
self.common_data.allowed_extensions.extend_allowed_extensions(VIDEO_FILES_EXTENSIONS);
if !self.common_data.allowed_extensions.using_custom_extensions() {
return true;
}
} }
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = let result = DirTraversalBuilder::new()
prepare_thread_handler_common(progress_sender, 0, 1, 0, CheckingMethod::None, self.common_data.tool_type); .group_by(|_fe| ())
.stop_receiver(stop_receiver)
.progress_sender(progress_sender)
.common_data(&self.common_data)
.max_stage(MAX_VIDEOS_STAGE)
.build()
.run();
while !folders_to_check.is_empty() { match result {
if check_if_stop_received(stop_receiver) { DirTraversalResult::SuccessFiles { grouped_file_entries, warnings } => {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); self.videos_to_check = grouped_file_entries
return false; .into_values()
} .flatten()
.map(|fe| (fe.path.to_string_lossy().to_string(), fe.into_videos_entry()))
let segments: Vec<_> = folders_to_check .collect();
.into_par_iter()
.map(|current_folder| {
let mut dir_result = vec![];
let mut warnings = vec![];
let mut fe_result = vec![];
let Some(read_dir) = common_read_dir(&current_folder, &mut warnings) else {
return (dir_result, warnings, fe_result);
};
// Check every sub folder/file/link etc.
for entry in read_dir {
let Ok(entry_data) = entry else {
continue;
};
let Ok(file_type) = entry_data.file_type() else {
continue;
};
if file_type.is_dir() {
check_folder_children(
&mut dir_result,
&mut warnings,
&entry_data,
self.common_data.recursive_search,
&self.common_data.directories,
&self.common_data.excluded_items,
);
} else if file_type.is_file() {
atomic_counter.fetch_add(1, Ordering::Relaxed);
self.add_video_file_entry(&entry_data, &mut fe_result, &mut warnings);
}
}
(dir_result, warnings, fe_result)
})
.collect();
let required_size = segments.iter().map(|(segment, _, _)| segment.len()).sum::<usize>();
folders_to_check = Vec::with_capacity(required_size);
// Process collected data
for (segment, warnings, fe_result) in segments {
folders_to_check.extend(segment);
self.common_data.text_messages.warnings.extend(warnings); self.common_data.text_messages.warnings.extend(warnings);
for (name, fe) in fe_result { debug!("check_files - Found {} video files.", self.videos_to_check.len());
self.videos_to_check.insert(name, fe); true
}
} }
}
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); DirTraversalResult::Stopped => false,
true
}
fn add_video_file_entry(&self, entry_data: &DirEntry, fe_result: &mut Vec<(String, FileEntry)>, warnings: &mut Vec<String>) {
if !self.common_data.allowed_extensions.check_if_entry_ends_with_extension(entry_data) {
return;
}
let current_file_name = entry_data.path();
if self.common_data.excluded_items.is_excluded(&current_file_name) {
return;
}
let current_file_name_str = current_file_name.to_string_lossy().to_string();
let Ok(metadata) = entry_data.metadata() else {
return;
};
// Checking files
if (self.common_data.minimal_file_size..=self.common_data.maximal_file_size).contains(&metadata.len()) {
let fe: FileEntry = FileEntry {
size: metadata.len(),
modified_date: get_modified_time(&metadata, warnings, &current_file_name, false),
path: current_file_name,
vhash: Default::default(),
error: String::new(),
};
fe_result.push((current_file_name_str, fe));
} }
} }
#[fun_time(message = "load_cache_at_start", level = "debug")] #[fun_time(message = "load_cache_at_start", level = "debug")]
fn load_cache_at_start(&mut self) -> (BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>, BTreeMap<String, FileEntry>) { fn load_cache_at_start(&mut self) -> (BTreeMap<String, VideosEntry>, BTreeMap<String, VideosEntry>, BTreeMap<String, VideosEntry>) {
let loaded_hash_map; let loaded_hash_map;
let mut records_already_cached: BTreeMap<String, FileEntry> = Default::default(); let mut records_already_cached: BTreeMap<String, VideosEntry> = Default::default();
let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default(); let mut non_cached_files_to_check: BTreeMap<String, VideosEntry> = Default::default();
if self.common_data.use_cache { if self.common_data.use_cache {
let (messages, loaded_items) = let (messages, loaded_items) =
load_cache_from_file_generalized_by_path::<FileEntry>(&get_similar_videos_cache_file(), self.get_delete_outdated_cache(), &self.videos_to_check); load_cache_from_file_generalized_by_path::<VideosEntry>(&get_similar_videos_cache_file(), self.get_delete_outdated_cache(), &self.videos_to_check);
self.get_text_messages_mut().extend_with_another_messages(messages); self.get_text_messages_mut().extend_with_another_messages(messages);
loaded_hash_map = loaded_items.unwrap_or_default(); loaded_hash_map = loaded_items.unwrap_or_default();
@ -268,7 +206,7 @@ impl SimilarVideos {
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type); prepare_thread_handler_common(progress_sender, 1, 1, non_cached_files_to_check.len(), CheckingMethod::None, self.common_data.tool_type);
let mut vec_file_entry: Vec<FileEntry> = non_cached_files_to_check let mut vec_file_entry: Vec<VideosEntry> = non_cached_files_to_check
.par_iter() .par_iter()
.map(|file_entry| { .map(|file_entry| {
atomic_counter.fetch_add(1, Ordering::Relaxed); atomic_counter.fetch_add(1, Ordering::Relaxed);
@ -293,14 +231,14 @@ impl SimilarVideos {
Some(file_entry) Some(file_entry)
}) })
.while_some() .while_some()
.collect::<Vec<FileEntry>>(); .collect::<Vec<VideosEntry>>();
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle); send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
// Just connect loaded results with already calculated hashes // Just connect loaded results with already calculated hashes
vec_file_entry.extend(records_already_cached.into_values()); vec_file_entry.extend(records_already_cached.into_values());
let mut hashmap_with_file_entries: HashMap<String, FileEntry> = Default::default(); let mut hashmap_with_file_entries: HashMap<String, VideosEntry> = Default::default();
let mut vector_of_hashes: Vec<VideoHash> = Vec::new(); let mut vector_of_hashes: Vec<VideoHash> = Vec::new();
for file_entry in &vec_file_entry { for file_entry in &vec_file_entry {
// 0 means that images was not hashed correctly, e.g. could be improperly // 0 means that images was not hashed correctly, e.g. could be improperly
@ -342,10 +280,10 @@ impl SimilarVideos {
} }
#[fun_time(message = "save_cache", level = "debug")] #[fun_time(message = "save_cache", level = "debug")]
fn save_cache(&mut self, vec_file_entry: Vec<FileEntry>, loaded_hash_map: BTreeMap<String, FileEntry>) { fn save_cache(&mut self, vec_file_entry: Vec<VideosEntry>, loaded_hash_map: BTreeMap<String, VideosEntry>) {
if self.common_data.use_cache { if self.common_data.use_cache {
// Must save all results to file, old loaded from file with all currently counted results // Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = loaded_hash_map; let mut all_results: BTreeMap<String, VideosEntry> = loaded_hash_map;
for file_entry in vec_file_entry { for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry); all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
} }
@ -356,11 +294,11 @@ impl SimilarVideos {
} }
#[fun_time(message = "match_groups_of_videos", level = "debug")] #[fun_time(message = "match_groups_of_videos", level = "debug")]
fn match_groups_of_videos(&mut self, vector_of_hashes: Vec<VideoHash>, hashmap_with_file_entries: &HashMap<String, FileEntry>) { fn match_groups_of_videos(&mut self, vector_of_hashes: Vec<VideoHash>, hashmap_with_file_entries: &HashMap<String, VideosEntry>) {
let match_group = vid_dup_finder_lib::search(vector_of_hashes, NormalizedTolerance::new(self.tolerance as f64 / 100.0f64)); let match_group = vid_dup_finder_lib::search(vector_of_hashes, NormalizedTolerance::new(self.tolerance as f64 / 100.0f64));
let mut collected_similar_videos: Vec<Vec<FileEntry>> = Default::default(); let mut collected_similar_videos: Vec<Vec<VideosEntry>> = Default::default();
for i in match_group { for i in match_group {
let mut temp_vector: Vec<FileEntry> = Vec::new(); let mut temp_vector: Vec<VideosEntry> = Vec::new();
let mut bt_size: BTreeSet<u64> = Default::default(); let mut bt_size: BTreeSet<u64> = Default::default();
for j in i.duplicates() { for j in i.duplicates() {
let file_entry = hashmap_with_file_entries.get(&j.to_string_lossy().to_string()).unwrap(); let file_entry = hashmap_with_file_entries.get(&j.to_string_lossy().to_string()).unwrap();
@ -397,7 +335,7 @@ impl SimilarVideos {
Some((files_from_referenced_folders.pop().unwrap(), normal_files)) Some((files_from_referenced_folders.pop().unwrap(), normal_files))
} }
}) })
.collect::<Vec<(FileEntry, Vec<FileEntry>)>>(); .collect::<Vec<(VideosEntry, Vec<VideosEntry>)>>();
} }
} }
@ -493,7 +431,7 @@ impl SimilarVideos {
self.tolerance = tolerance; self.tolerance = tolerance;
} }
pub const fn get_similar_videos(&self) -> &Vec<Vec<FileEntry>> { pub const fn get_similar_videos(&self) -> &Vec<Vec<VideosEntry>> {
&self.similar_vectors &self.similar_vectors
} }
@ -501,7 +439,7 @@ impl SimilarVideos {
&self.information &self.information
} }
pub fn get_similar_videos_referenced(&self) -> &Vec<(FileEntry, Vec<FileEntry>)> { pub fn get_similar_videos_referenced(&self) -> &Vec<(VideosEntry, Vec<VideosEntry>)> {
&self.similar_referenced_vectors &self.similar_referenced_vectors
} }

14
czkawka_core/src/temporary.rs
View file

@ -12,7 +12,7 @@ use rayon::prelude::*;
use serde::Serialize; use serde::Serialize;
use crate::common::{check_folder_children, check_if_stop_received, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads}; use crate::common::{check_folder_children, check_if_stop_received, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_dir_traversal::{common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_dir_traversal::{common_read_dir, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod}; use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::*; use crate::common_traits::*;
@ -142,16 +142,18 @@ impl Temporary {
pub fn get_file_entry(&self, atomic_counter: &Arc<AtomicUsize>, entry_data: &DirEntry, warnings: &mut Vec<String>) -> Option<FileEntry> { pub fn get_file_entry(&self, atomic_counter: &Arc<AtomicUsize>, entry_data: &DirEntry, warnings: &mut Vec<String>) -> Option<FileEntry> {
atomic_counter.fetch_add(1, Ordering::Relaxed); atomic_counter.fetch_add(1, Ordering::Relaxed);
let file_name_lowercase = get_lowercase_name(entry_data, warnings)?;
if !TEMP_EXTENSIONS.iter().any(|f| file_name_lowercase.ends_with(f)) {
return None;
}
let current_file_name = entry_data.path(); let current_file_name = entry_data.path();
if self.common_data.excluded_items.is_excluded(&current_file_name) { if self.common_data.excluded_items.is_excluded(&current_file_name) {
return None; return None;
} }
let file_name = entry_data.file_name();
let file_name_ascii_lowercase = file_name.to_ascii_lowercase();
let file_name_lowercase = file_name_ascii_lowercase.to_string_lossy();
if !TEMP_EXTENSIONS.iter().any(|f| file_name_lowercase.ends_with(f)) {
return None;
}
let Ok(metadata) = entry_data.metadata() else { let Ok(metadata) = entry_data.metadata() else {
return None; return None;
}; };

5
czkawka_gui/Cargo.toml
View file

@ -12,12 +12,13 @@ repository = "https://github.com/qarmin/czkawka"
[dependencies] [dependencies]
gdk4 = "0.7" gdk4 = "0.7"
glib = "0.18" glib = "0.18"
gtk4 = { version = "0.7", default-features = false, features = ["v4_6"] }
humansize = "2.1" humansize = "2.1"
chrono = "0.4.31" chrono = "0.4.31"
# Used for sending stop signal across threads # Used for sending stop signal across threads
crossbeam-channel = "0.5.8" crossbeam-channel = "0.5"
# For saving/loading config files to specific directories # For saving/loading config files to specific directories
directories-next = "2.0" directories-next = "2.0"
@ -49,9 +50,9 @@ once_cell = "1.19"
log = "0.4.20" log = "0.4.20"
handsome_logger = "0.8" handsome_logger = "0.8"
fun_time = { version = "0.3", features = ["log"] } fun_time = { version = "0.3", features = ["log"] }
rayon = "1.8"
czkawka_core = { path = "../czkawka_core", version = "6.1.0", features = [] } czkawka_core = { path = "../czkawka_core", version = "6.1.0", features = [] }
gtk4 = { version = "0.7", default-features = false, features = ["v4_6"] }
[target.'cfg(windows)'.dependencies] [target.'cfg(windows)'.dependencies]
winapi = { version = "0.3.9", features = ["combaseapi", "objbase", "shobjidl_core", "windef", "winerror", "wtypesbase", "winuser"] } winapi = { version = "0.3.9", features = ["combaseapi", "objbase", "shobjidl_core", "windef", "winerror", "wtypesbase", "winuser"] }

52
czkawka_gui/src/compute_results.rs
View file

@ -8,13 +8,15 @@ use glib::Receiver;
use gtk4::prelude::*; use gtk4::prelude::*;
use gtk4::{Entry, ListStore, TextView, TreeView, Widget}; use gtk4::{Entry, ListStore, TextView, TreeView, Widget};
use humansize::{format_size, BINARY}; use humansize::{format_size, BINARY};
use rayon::prelude::*;
use czkawka_core::bad_extensions::BadExtensions; use czkawka_core::bad_extensions::BadExtensions;
use czkawka_core::big_file::BigFile; use czkawka_core::big_file::BigFile;
use czkawka_core::broken_files::BrokenFiles; use czkawka_core::broken_files::BrokenFiles;
use czkawka_core::common::{split_path, split_path_compare}; use czkawka_core::common::{split_path, split_path_compare};
use czkawka_core::common_dir_traversal::{CheckingMethod, FileEntry}; use czkawka_core::common_dir_traversal::CheckingMethod;
use czkawka_core::common_tool::CommonData; use czkawka_core::common_tool::CommonData;
use czkawka_core::common_traits::ResultEntry;
use czkawka_core::duplicate::DuplicateFinder; use czkawka_core::duplicate::DuplicateFinder;
use czkawka_core::empty_files::EmptyFiles; use czkawka_core::empty_files::EmptyFiles;
use czkawka_core::empty_folder::EmptyFolder; use czkawka_core::empty_folder::EmptyFolder;
@ -22,7 +24,7 @@ use czkawka_core::invalid_symlinks::InvalidSymlinks;
use czkawka_core::localizer_core::generate_translation_hashmap; use czkawka_core::localizer_core::generate_translation_hashmap;
use czkawka_core::same_music::{MusicSimilarity, SameMusic}; use czkawka_core::same_music::{MusicSimilarity, SameMusic};
use czkawka_core::similar_images; use czkawka_core::similar_images;
use czkawka_core::similar_images::SimilarImages; use czkawka_core::similar_images::{ImagesEntry, SimilarImages};
use czkawka_core::similar_videos::SimilarVideos; use czkawka_core::similar_videos::SimilarVideos;
use czkawka_core::temporary::Temporary; use czkawka_core::temporary::Temporary;
@ -264,7 +266,7 @@ fn computer_bad_extensions(
// Sort // Sort
let mut vector = vector.clone(); let mut vector = vector.clone();
vector.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vector.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
for file_entry in vector { for file_entry in vector {
let (directory, file) = split_path(&file_entry.path); let (directory, file) = split_path(&file_entry.path);
@ -336,7 +338,7 @@ fn computer_broken_files(
// Sort // Sort
let mut vector = vector.clone(); let mut vector = vector.clone();
vector.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vector.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
for file_entry in vector { for file_entry in vector {
let (directory, file) = split_path(&file_entry.path); let (directory, file) = split_path(&file_entry.path);
@ -407,7 +409,7 @@ fn computer_invalid_symlinks(
for file_entry in vector { for file_entry in vector {
let (directory, file) = split_path(&file_entry.path); let (directory, file) = split_path(&file_entry.path);
let symlink_info = file_entry.symlink_info.clone().expect("invalid traversal result"); let symlink_info = file_entry.symlink_info;
let values: [(u32, &dyn ToValue); COLUMNS_NUMBER] = [ let values: [(u32, &dyn ToValue); COLUMNS_NUMBER] = [
(ColumnsInvalidSymlinks::SelectionButton as u32, &false), (ColumnsInvalidSymlinks::SelectionButton as u32, &false),
(ColumnsInvalidSymlinks::Name as u32, &file), (ColumnsInvalidSymlinks::Name as u32, &file),
@ -499,7 +501,7 @@ fn computer_same_music(
// Sort // Sort
let vec_file_entry = if vec_file_entry.len() >= 2 { let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone(); let mut vec_file_entry = vec_file_entry.clone();
vec_file_entry.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vec_file_entry.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
vec_file_entry vec_file_entry
} else { } else {
vec_file_entry.clone() vec_file_entry.clone()
@ -551,7 +553,7 @@ fn computer_same_music(
// Sort // Sort
let vec_file_entry = if vec_file_entry.len() >= 2 { let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone(); let mut vec_file_entry = vec_file_entry.clone();
vec_file_entry.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vec_file_entry.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
vec_file_entry vec_file_entry
} else { } else {
vec_file_entry.clone() vec_file_entry.clone()
@ -657,7 +659,7 @@ fn computer_similar_videos(
// Sort // Sort
let vec_file_entry = if vec_file_entry.len() >= 2 { let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone(); let mut vec_file_entry = vec_file_entry.clone();
vec_file_entry.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vec_file_entry.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
vec_file_entry vec_file_entry
} else { } else {
vec_file_entry.clone() vec_file_entry.clone()
@ -676,7 +678,7 @@ fn computer_similar_videos(
// Sort // Sort
let vec_file_entry = if vec_file_entry.len() >= 2 { let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone(); let mut vec_file_entry = vec_file_entry.clone();
vec_file_entry.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vec_file_entry.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
vec_file_entry vec_file_entry
} else { } else {
vec_file_entry.clone() vec_file_entry.clone()
@ -749,13 +751,13 @@ fn computer_similar_images(
let list_store = get_list_store(tree_view); let list_store = get_list_store(tree_view);
if sf.get_use_reference() { if sf.get_use_reference() {
let vec_struct_similar: &Vec<(similar_images::FileEntry, Vec<similar_images::FileEntry>)> = sf.get_similar_images_referenced(); let vec_struct_similar: &Vec<(ImagesEntry, Vec<ImagesEntry>)> = sf.get_similar_images_referenced();
for (base_file_entry, vec_file_entry) in vec_struct_similar { for (base_file_entry, vec_file_entry) in vec_struct_similar {
// Sort // Sort
let vec_file_entry = if vec_file_entry.len() >= 2 { let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone(); let mut vec_file_entry = vec_file_entry.clone();
// Use comparison by similarity, because it is more important that path here // Use comparison by similarity, because it is more important that path here
vec_file_entry.sort_unstable_by_key(|e| e.similarity); vec_file_entry.par_sort_unstable_by_key(|e| e.similarity);
vec_file_entry vec_file_entry
} else { } else {
vec_file_entry.clone() vec_file_entry.clone()
@ -798,7 +800,7 @@ fn computer_similar_images(
let vec_file_entry = if vec_file_entry.len() >= 2 { let vec_file_entry = if vec_file_entry.len() >= 2 {
let mut vec_file_entry = vec_file_entry.clone(); let mut vec_file_entry = vec_file_entry.clone();
// Use comparison by similarity, because it is more important that path here // Use comparison by similarity, because it is more important that path here
vec_file_entry.sort_unstable_by_key(|e| e.similarity); vec_file_entry.par_sort_unstable_by_key(|e| e.similarity);
vec_file_entry vec_file_entry
} else { } else {
vec_file_entry.clone() vec_file_entry.clone()
@ -876,7 +878,7 @@ fn computer_temporary_files(
// Sort // TODO maybe simplify this via common file entry // Sort // TODO maybe simplify this via common file entry
let mut vector = vector.clone(); let mut vector = vector.clone();
vector.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vector.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
for file_entry in vector { for file_entry in vector {
let (directory, file) = split_path(&file_entry.path); let (directory, file) = split_path(&file_entry.path);
@ -1080,7 +1082,7 @@ fn computer_empty_folders(
let hashmap = ef.get_empty_folder_list(); let hashmap = ef.get_empty_folder_list();
let mut vector = hashmap.values().collect::<Vec<_>>(); let mut vector = hashmap.values().collect::<Vec<_>>();
vector.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vector.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
for fe in vector { for fe in vector {
let (directory, file) = split_path(&fe.path); let (directory, file) = split_path(&fe.path);
@ -1325,19 +1327,27 @@ fn computer_duplicate_finder(
} }
} }
fn vector_sort_unstable_entry_by_path(vector: &[FileEntry]) -> Vec<FileEntry> { fn vector_sort_unstable_entry_by_path<T>(vector: &[T]) -> Vec<T>
where
T: ResultEntry + Clone,
T: std::marker::Send,
{
if vector.len() >= 2 { if vector.len() >= 2 {
let mut vector = vector.to_owned(); let mut vector = vector.to_vec();
vector.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vector.par_sort_unstable_by(|a, b| split_path_compare(a.get_path(), b.get_path()));
vector vector
} else { } else {
vector.to_owned() vector.to_vec()
} }
} }
fn vector_sort_simple_unstable_entry_by_path(vector: &[FileEntry]) -> Vec<FileEntry> { fn vector_sort_simple_unstable_entry_by_path<T>(vector: &[T]) -> Vec<T>
let mut vector = vector.to_owned(); where
vector.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); T: ResultEntry + Clone,
T: std::marker::Send,
{
let mut vector = vector.to_vec();
vector.par_sort_unstable_by(|a, b| split_path_compare(a.get_path(), b.get_path()));
vector vector
} }

7
czkawka_gui/src/connect_things/connect_button_compare.rs
View file

@ -353,8 +353,11 @@ fn generate_cache_for_results(vector_with_path: Vec<(String, String, TreePath)>)
let big_img = Image::new(); let big_img = Image::new();
let mut pixbuf = get_pixbuf_from_dynamic_image(&DynamicImage::new_rgb8(1, 1)).unwrap(); let mut pixbuf = get_pixbuf_from_dynamic_image(&DynamicImage::new_rgb8(1, 1)).unwrap();
let name_lowercase = name.to_lowercase(); let extension_lowercase = full_path.split('.').last().map(str::to_lowercase);
let is_heic = HEIC_EXTENSIONS.iter().any(|extension| name_lowercase.ends_with(extension)); let is_heic = match extension_lowercase {
Some(extension) => HEIC_EXTENSIONS.iter().any(|e| e == &extension),
None => false,
};
if is_heic { if is_heic {
#[allow(clippy::never_loop)] #[allow(clippy::never_loop)]

4
czkawka_gui/src/connect_things/connect_settings.rs
View file

@ -178,7 +178,7 @@ pub fn connect_settings(gui_data: &GuiData) {
FilterType::Triangle, FilterType::Triangle,
] { ] {
for hash_alg in &[HashAlg::Blockhash, HashAlg::Gradient, HashAlg::DoubleGradient, HashAlg::VertGradient, HashAlg::Mean] { for hash_alg in &[HashAlg::Blockhash, HashAlg::Gradient, HashAlg::DoubleGradient, HashAlg::VertGradient, HashAlg::Mean] {
let (mut messages, loaded_items) = load_cache_from_file_generalized_by_path::<czkawka_core::similar_images::FileEntry>( let (mut messages, loaded_items) = load_cache_from_file_generalized_by_path::<czkawka_core::similar_images::ImagesEntry>(
&get_similar_images_cache_file(hash_size, hash_alg, image_filter), &get_similar_images_cache_file(hash_size, hash_alg, image_filter),
true, true,
&Default::default(), &Default::default(),
@ -214,7 +214,7 @@ pub fn connect_settings(gui_data: &GuiData) {
dialog.connect_response(move |dialog, response_type| { dialog.connect_response(move |dialog, response_type| {
if response_type == ResponseType::Ok { if response_type == ResponseType::Ok {
let (mut messages, loaded_items) = let (mut messages, loaded_items) =
load_cache_from_file_generalized_by_path::<czkawka_core::similar_videos::FileEntry>(&get_similar_videos_cache_file(), true, &Default::default()); load_cache_from_file_generalized_by_path::<czkawka_core::similar_videos::VideosEntry>(&get_similar_videos_cache_file(), true, &Default::default());
if let Some(cache_entries) = loaded_items { if let Some(cache_entries) = loaded_items {
let save_messages = save_cache_to_file_generalized(&get_similar_videos_cache_file(), &cache_entries, false, 0); let save_messages = save_cache_to_file_generalized(&get_similar_videos_cache_file(), &cache_entries, false, 0);

6
czkawka_gui/src/initialize_gui.rs
View file

@ -497,9 +497,9 @@ fn show_preview(
let is_heic; let is_heic;
if let Some(extension) = Path::new(&name).extension() { if let Some(extension) = Path::new(&name).extension() {
let extension = format!(".{}", extension.to_string_lossy().to_lowercase()); let extension_lowercase = extension.to_string_lossy().to_lowercase();
is_heic = HEIC_EXTENSIONS.contains(&extension.as_str()); is_heic = HEIC_EXTENSIONS.contains(&extension_lowercase.as_str());
if !RAW_IMAGE_EXTENSIONS.contains(&extension.as_str()) && !IMAGE_RS_EXTENSIONS.contains(&extension.as_str()) && !is_heic { if !RAW_IMAGE_EXTENSIONS.contains(&extension_lowercase.as_str()) && !IMAGE_RS_EXTENSIONS.contains(&extension_lowercase.as_str()) && !is_heic {
break 'dir; break 'dir;
} }
} else { } else {

2
krokiet/Cargo.toml
View file

@ -24,7 +24,7 @@ rand = "0.8"
czkawka_core = { version = "6.1.0", path = "../czkawka_core" } czkawka_core = { version = "6.1.0", path = "../czkawka_core" }
chrono = "0.4.31" chrono = "0.4.31"
open = "5.0" open = "5.0"
crossbeam-channel = "0.5.8" crossbeam-channel = "0.5"
handsome_logger = "0.8" handsome_logger = "0.8"
rfd = { version = "0.12", default-features = false, features = ["xdg-portal"] } rfd = { version = "0.12", default-features = false, features = ["xdg-portal"] }
home = "0.5" home = "0.5"

14
krokiet/src/connect_scan.rs
View file

@ -7,11 +7,11 @@ use czkawka_core::common_dir_traversal::{FileEntry, ProgressData};
use czkawka_core::common_tool::CommonData; use czkawka_core::common_tool::CommonData;
use czkawka_core::common_traits::ResultEntry; use czkawka_core::common_traits::ResultEntry;
use czkawka_core::empty_files::EmptyFiles; use czkawka_core::empty_files::EmptyFiles;
use czkawka_core::empty_folder::EmptyFolder; use czkawka_core::empty_folder::{EmptyFolder, FolderEntry};
use czkawka_core::empty_folder::FolderEntry;
use czkawka_core::similar_images; use czkawka_core::similar_images;
use czkawka_core::similar_images::SimilarImages; use czkawka_core::similar_images::SimilarImages;
use humansize::{format_size, BINARY}; use humansize::{format_size, BINARY};
use rayon::prelude::*;
use slint::{ComponentHandle, ModelRc, SharedString, VecModel, Weak}; use slint::{ComponentHandle, ModelRc, SharedString, VecModel, Weak};
use std::rc::Rc; use std::rc::Rc;
use std::thread; use std::thread;
@ -75,10 +75,10 @@ fn scan_similar_images(a: Weak<MainWindow>, progress_sender: Sender<ProgressData
let messages = finder.get_text_messages().create_messages_text(); let messages = finder.get_text_messages().create_messages_text();
for vec_fe in &mut vector { for vec_fe in &mut vector {
vec_fe.sort_unstable_by_key(|e| e.similarity); vec_fe.par_sort_unstable_by_key(|e| e.similarity);
} }
let hash_size = finder.hash_size; let hash_size = custom_settings.similar_images_sub_hash_size;
a.upgrade_in_event_loop(move |app| { a.upgrade_in_event_loop(move |app| {
write_similar_images_results(&app, vector, messages, hash_size); write_similar_images_results(&app, vector, messages, hash_size);
@ -86,7 +86,7 @@ fn scan_similar_images(a: Weak<MainWindow>, progress_sender: Sender<ProgressData
}) })
.unwrap(); .unwrap();
} }
fn write_similar_images_results(app: &MainWindow, vector: Vec<Vec<similar_images::FileEntry>>, messages: String, hash_size: u8) { fn write_similar_images_results(app: &MainWindow, vector: Vec<Vec<similar_images::ImagesEntry>>, messages: String, hash_size: u8) {
let items_found = vector.len(); let items_found = vector.len();
let items = Rc::new(VecModel::default()); let items = Rc::new(VecModel::default());
for vec_fe in vector { for vec_fe in vector {
@ -121,7 +121,7 @@ fn scan_empty_files(a: Weak<MainWindow>, progress_sender: Sender<ProgressData>,
let mut vector = finder.get_empty_files().clone(); let mut vector = finder.get_empty_files().clone();
let messages = finder.get_text_messages().create_messages_text(); let messages = finder.get_text_messages().create_messages_text();
vector.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vector.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
a.upgrade_in_event_loop(move |app| { a.upgrade_in_event_loop(move |app| {
write_empty_files_results(&app, vector, messages); write_empty_files_results(&app, vector, messages);
@ -158,7 +158,7 @@ fn scan_empty_folders(a: Weak<MainWindow>, progress_sender: Sender<ProgressData>
let mut vector = finder.get_empty_folder_list().values().cloned().collect::<Vec<_>>(); let mut vector = finder.get_empty_folder_list().values().cloned().collect::<Vec<_>>();
let messages = finder.get_text_messages().create_messages_text(); let messages = finder.get_text_messages().create_messages_text();
vector.sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path())); vector.par_sort_unstable_by(|a, b| split_path_compare(a.path.as_path(), b.path.as_path()));
a.upgrade_in_event_loop(move |app| { a.upgrade_in_event_loop(move |app| {
write_empty_folders_results(&app, vector, messages); write_empty_folders_results(&app, vector, messages);

5
krokiet/src/connect_show_preview.rs
View file

@ -48,10 +48,9 @@ fn load_image(image_path: &Path) -> Option<(Duration, image::DynamicImage)> {
} }
let image_name = image_path.to_string_lossy().to_string(); let image_name = image_path.to_string_lossy().to_string();
let image_extension = image_path.extension()?.to_string_lossy().to_lowercase(); let image_extension = image_path.extension()?.to_string_lossy().to_lowercase();
let extension_with_dot = format!(".{}", image_extension);
let is_raw_image = RAW_IMAGE_EXTENSIONS.contains(&extension_with_dot.as_str()); let is_raw_image = RAW_IMAGE_EXTENSIONS.contains(&image_extension.as_str());
let is_normal_image = IMAGE_RS_EXTENSIONS.contains(&extension_with_dot.as_str()); let is_normal_image = IMAGE_RS_EXTENSIONS.contains(&image_extension.as_str());
if !is_raw_image && !is_normal_image { if !is_raw_image && !is_normal_image {
return None; return None;

3
krokiet/src/settings.rs
View file

@ -129,7 +129,7 @@ pub fn connect_changing_settings_preset(app: &MainWindow) {
app.global::<Callabler>().on_changed_settings_preset(move || { app.global::<Callabler>().on_changed_settings_preset(move || {
let app = a.upgrade().unwrap(); let app = a.upgrade().unwrap();
let current_item = app.global::<Settings>().get_settings_preset_idx(); let current_item = app.global::<Settings>().get_settings_preset_idx();
let loaded_data = load_data_from_file::<SettingsCustom>(get_config_file(current_item)); let loaded_data = load_data_from_file::<SettingsCustom>(get_config_file(current_item + 1));
match loaded_data { match loaded_data {
Ok(loaded_data) => { Ok(loaded_data) => {
set_settings_to_gui(&app, &loaded_data); set_settings_to_gui(&app, &loaded_data);
@ -138,6 +138,7 @@ pub fn connect_changing_settings_preset(app: &MainWindow) {
Err(e) => { Err(e) => {
set_settings_to_gui(&app, &SettingsCustom::default()); set_settings_to_gui(&app, &SettingsCustom::default());
app.set_text_summary_text(format!("Cannot change and load preset {} - reason {e}", current_item + 1).into()); app.set_text_summary_text(format!("Cannot change and load preset {} - reason {e}", current_item + 1).into());
error!("Failed to change preset - {e}, using default instead");
} }
} }
}); });