Update BKtree and blake3 and add new grouping algorithm (#405)
* Update BKtree and blake3 * Add new algorithm for similar images * Improve algorithm of grouping similar images
This commit is contained in:
Rafał Mikrut
GitHub
parent
da76f75e11
commit
909967dcd8
|
|
@ -59,9 +59,9 @@ checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
|
|||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.5.2"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
|
||||
checksum = "be4dc07131ffa69b8072d35f5007352af944213cde02545e2103680baed38fcd"
|
||||
|
||||
[[package]]
|
||||
name = "atk"
|
||||
|
|
@ -156,22 +156,25 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
|||
|
||||
[[package]]
|
||||
name = "bk-tree"
|
||||
version = "0.3.0"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5488039ea2c6de8668351415e39a0218a8955bffadcff0cf01d1293a20854584"
|
||||
checksum = "6121f6e107e94d717b5ca2631d56e7c2ed1542a21b2eb87b4bda1d6c1420ef3f"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"triple_accel",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blake3"
|
||||
version = "0.3.8"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b64485778c4f16a6a5a9d335e80d449ac6c70cdd6a06d2af18a6f6f775a125b3"
|
||||
checksum = "dcd555c66291d5f836dbb6883b48660ece810fe25a31f3bdfb911945dff2691f"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"arrayvec",
|
||||
"cc",
|
||||
"cfg-if 0.1.10",
|
||||
"cfg-if",
|
||||
"constant_time_eq",
|
||||
"crypto-mac",
|
||||
"digest",
|
||||
]
|
||||
|
||||
|
|
@ -270,19 +273,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cfg-expr"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70a1d12766bbdd5d44caab5df04a9bffec9cd855a1b44b15de5665d70c085f94"
|
||||
checksum = "b412e83326147c2bb881f8b40edfbf9905b9b8abaebd0e47ca190ba62fda8f0e"
|
||||
dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
|
|
@ -412,7 +409,7 @@ version = "1.2.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -421,17 +418,17 @@ version = "0.5.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
|
||||
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
|
@ -442,7 +439,7 @@ version = "0.9.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"lazy_static",
|
||||
"memoffset",
|
||||
|
|
@ -455,20 +452,10 @@ version = "0.8.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-mac"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "czkawka_cli"
|
||||
version = "3.1.0"
|
||||
|
|
@ -590,7 +577,7 @@ version = "2.0.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"dirs-sys-next",
|
||||
]
|
||||
|
||||
|
|
@ -627,7 +614,7 @@ version = "1.0.20"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"crc32fast",
|
||||
"libc",
|
||||
"miniz_oxide 0.4.4",
|
||||
|
|
@ -807,7 +794,7 @@ version = "0.2.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
|
@ -1066,7 +1053,7 @@ version = "0.1.10"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1100,9 +1087,9 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
|
|||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.22"
|
||||
version = "0.1.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd"
|
||||
checksum = "f5ca711fd837261e14ec9e674f092cbb931d3fa1482b017ae59328ddc6f3212b"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
|
@ -1118,9 +1105,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.51"
|
||||
version = "0.3.52"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062"
|
||||
checksum = "ce791b7ca6638aae45be056e068fc756d871eb3b3b10b8efa62d1c9cec616752"
|
||||
dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
|
@ -1160,7 +1147,7 @@ version = "0.7.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f84d96438c15fcd6c3f244c8fce01d1e2b9c6b5623e9c711dc9286d8fc92d6a"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
|
|
@ -1179,7 +1166,7 @@ version = "0.4.14"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1325,7 +1312,7 @@ checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a"
|
|||
dependencies = [
|
||||
"bitflags",
|
||||
"cc",
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
|
||||
|
|
@ -1413,9 +1400,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "num_enum"
|
||||
version = "0.5.2"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5adf0198d427ee515335639f275e806ca01acf9f07d7cf14bb36a10532a6169"
|
||||
checksum = "ee2c8fd66061a707503d515639b8af10fd3807a5b5ee6959f7ff1bd303634bd5"
|
||||
dependencies = [
|
||||
"derivative",
|
||||
"num_enum_derive",
|
||||
|
|
@ -1423,9 +1410,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "num_enum_derive"
|
||||
version = "0.5.2"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1def5a3f69d4707d8a040b12785b98029a39e8c610ae685c7f6265669767482"
|
||||
checksum = "474fd1d096da3ad17084694eebed40ba09c4a36c5255cd772bd8b98859cc562e"
|
||||
dependencies = [
|
||||
"proc-macro-crate 1.0.0",
|
||||
"proc-macro2",
|
||||
|
|
@ -1523,7 +1510,7 @@ version = "0.8.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
|
|
@ -1728,9 +1715,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.9"
|
||||
version = "0.2.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
|
||||
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
|
@ -1860,18 +1847,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.126"
|
||||
version = "1.0.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
|
||||
checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.126"
|
||||
version = "1.0.127"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
|
||||
checksum = "a024926d3432516606328597e0f224a51355a493b49fdd67e9209187cbe55ecc"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -1973,12 +1960,6 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.74"
|
||||
|
|
@ -2014,7 +1995,7 @@ version = "3.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"rand",
|
||||
"redox_syscall",
|
||||
|
|
@ -2122,6 +2103,12 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "triple_accel"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "622b09ce2fe2df4618636fb92176d205662f59803f39e70d1c333393082de96c"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.13.0"
|
||||
|
|
@ -2189,19 +2176,19 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.74"
|
||||
version = "0.2.75"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
|
||||
checksum = "b608ecc8f4198fe8680e2ed18eccab5f0cd4caaf3d83516fa5fb2e927fda2586"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"wasm-bindgen-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-backend"
|
||||
version = "0.2.74"
|
||||
version = "0.2.75"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
|
||||
checksum = "580aa3a91a63d23aac5b6b267e2d13cb4f363e31dce6c352fca4752ae12e479f"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"lazy_static",
|
||||
|
|
@ -2214,9 +2201,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.74"
|
||||
version = "0.2.75"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
|
||||
checksum = "171ebf0ed9e1458810dfcb31f2e766ad6b3a89dbda42d8901f2b268277e5f09c"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
|
|
@ -2224,9 +2211,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.74"
|
||||
version = "0.2.75"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
|
||||
checksum = "6c2657dd393f03aa2a659c25c6ae18a13a4048cebd220e147933ea837efc589f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -2237,15 +2224,15 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.74"
|
||||
version = "0.2.75"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"
|
||||
checksum = "2e0c4a743a309662d45f4ede961d7afa4ba4131a59a639f29b0069c3798bbcc2"
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.51"
|
||||
version = "0.3.52"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582"
|
||||
checksum = "01c70a82d842c9979078c772d4a1344685045f1a5628f677c2b2eab4dd7d2696"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ You can help by creating:
|
|||
You can also help by doing different things:
|
||||
- Creating text articles - [LinuxUprising](https://www.linuxuprising.com/2021/03/find-and-remove-duplicate-files-similar.html) or [Ubunlog](https://ubunlog.com/en/czkawka-finds-and-removes-empty-and-broken-duplicate-files/)
|
||||
- Adding Czkawka to repositories - [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=czkawka&branch=edge) or [NixOS](https://github.com/NixOS/nixpkgs/pull/116441) or [OpenMandriva](https://github.com/OpenMandrivaAssociation/czkawka)
|
||||
- Creating videos - [Tutorial Spanish 1](https://www.youtube.com/watch?v=tALYBsJAYwE) or [Tutorial Spanish 2](https://www.youtube.com/watch?v=V9x-pHJRmKY)
|
||||
- Creating videos - [First Video](https://www.youtube.com/watch?v=CWlRiTD4vDc) or [Tutorial Spanish](https://www.youtube.com/watch?v=V9x-pHJRmKY)
|
||||
- Recommending it to others
|
||||
|
||||
## Name
|
||||
|
|
@ -159,5 +159,11 @@ The program is completely free to use.
|
|||
|
||||
"Gratis to uczciwa cena" - "Free is a fair price"
|
||||
|
||||
## Thanks
|
||||
|
||||
Big thanks to Pádraig Brady, creator of fantastic FSlint, because without his work I wouldn't create this tool.
|
||||
|
||||
Thanks also to all the people who create patches for this program, make it available on other systems, create videos, articles about it etc.
|
||||
|
||||
## Donations
|
||||
If you are using the app, I would appreciate a donation for its further development, which can be done [here](https://github.com/sponsors/qarmin).
|
||||
|
|
|
|||
|
|
@ -289,10 +289,10 @@ fn parse_delete_method(src: &str) -> Result<DeleteMethod, &'static str> {
|
|||
// TODO For now it looks different
|
||||
fn parse_similar_images_similarity(src: &str) -> Result<Similarity, &'static str> {
|
||||
match src.to_ascii_lowercase().replace('_', "").as_str() {
|
||||
"minimal" => Ok(Similarity::Similar(5)),
|
||||
"verysmall" => Ok(Similarity::Similar(4)),
|
||||
"small" => Ok(Similarity::Similar(3)),
|
||||
"medium" => Ok(Similarity::Similar(2)),
|
||||
"minimal" => Ok(Similarity::Similar(12)),
|
||||
"verysmall" => Ok(Similarity::Similar(8)),
|
||||
"small" => Ok(Similarity::Similar(5)),
|
||||
"medium" => Ok(Similarity::Similar(3)),
|
||||
"high" => Ok(Similarity::Similar(1)),
|
||||
"veryhigh" => Ok(Similarity::Similar(0)),
|
||||
_ => Err("Couldn't parse the delete method (allowed: verysmall, small, medium, high, veryhigh)"),
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ directories-next = "2.0.0"
|
|||
|
||||
# Needed by similar images
|
||||
img_hash = "3.2.0"
|
||||
bk-tree = "0.3.0" # Do not update. Updating broke of searching for similar hashes
|
||||
bk-tree = "0.4.0"
|
||||
image = "0.23.14"
|
||||
hamming = "0.1.3"
|
||||
|
||||
|
|
@ -35,7 +35,7 @@ zip = "0.5.13"
|
|||
rodio = { version = "0.14.0", optional = true }
|
||||
|
||||
# Hashes
|
||||
blake3 = "0.3.8"
|
||||
blake3 = "1.0.0"
|
||||
crc32fast = "1.2.1"
|
||||
xxhash-rust = { version = "0.8.2", features = ["xxh3"] }
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ use humansize::{file_size_opts as options, FileSize};
|
|||
use image::GenericImageView;
|
||||
use img_hash::HasherConfig;
|
||||
use rayon::prelude::*;
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::OpenOptions;
|
||||
use std::fs::{File, Metadata};
|
||||
use std::io::Write;
|
||||
|
|
@ -38,9 +38,11 @@ pub struct ProgressData {
|
|||
#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Debug)]
|
||||
pub enum Similarity {
|
||||
None,
|
||||
Similar(u64),
|
||||
Similar(u32),
|
||||
}
|
||||
|
||||
const MAX_SIMILARITY: u32 = 12;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FileEntry {
|
||||
pub path: PathBuf,
|
||||
|
|
@ -55,14 +57,13 @@ pub struct FileEntry {
|
|||
struct Hamming;
|
||||
|
||||
impl bk_tree::Metric<Node> for Hamming {
|
||||
fn distance(&self, a: &Node, b: &Node) -> u64 {
|
||||
hamming::distance_fast(a, b).unwrap() as u64
|
||||
fn distance(&self, a: &Node, b: &Node) -> u32 {
|
||||
hamming::distance_fast(a, b).unwrap() as u32
|
||||
}
|
||||
|
||||
// // TODO Probably needs to be implemented
|
||||
// fn threshold_distance(&self, _a: &Node, _b: &Node, _threshold: u32) -> Option<u32> {
|
||||
// None
|
||||
// }
|
||||
fn threshold_distance(&self, a: &Node, b: &Node, _threshold: u32) -> Option<u32> {
|
||||
Some(self.distance(a, b))
|
||||
}
|
||||
}
|
||||
|
||||
/// Struct to store most basics info about all folder
|
||||
|
|
@ -454,61 +455,69 @@ impl SimilarImages {
|
|||
Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - saving data to files".to_string());
|
||||
let hash_map_modification = SystemTime::now();
|
||||
|
||||
let similarity: u64 = match self.similarity {
|
||||
let similarity: u32 = match self.similarity {
|
||||
Similarity::Similar(k) => k,
|
||||
_ => panic!(),
|
||||
};
|
||||
|
||||
// TODO
|
||||
// Now is A is similar to B with VeryHigh and C with Medium
|
||||
// And D is similar with C with High
|
||||
// And Similarity is set to Medium(or lower)
|
||||
// And A is checked before D
|
||||
// Then C is shown that is similar group A, not D
|
||||
|
||||
// TODO
|
||||
// Maybe also add here progress report
|
||||
|
||||
let mut new_vector: Vec<Vec<FileEntry>> = Vec::new();
|
||||
let mut non_cached_files_to_check = self.image_hashes.clone();
|
||||
for (hash, vec_file_entry) in &self.image_hashes {
|
||||
let mut collected_similar_images: BTreeMap<Node, Vec<FileEntry>> = Default::default();
|
||||
|
||||
let mut available_hashes = self.image_hashes.clone();
|
||||
let mut this_time_check_hashes;
|
||||
let mut master_of_group: BTreeSet<Node> = Default::default(); // Lista wszystkich głównych hashy, które odpowiadają za porównywanie
|
||||
|
||||
for current_similarity in 0..=MAX_SIMILARITY {
|
||||
this_time_check_hashes = available_hashes.clone();
|
||||
|
||||
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
|
||||
return false;
|
||||
}
|
||||
if !non_cached_files_to_check.contains_key(hash) {
|
||||
continue;
|
||||
}
|
||||
non_cached_files_to_check.remove(hash);
|
||||
|
||||
let vector_with_found_similar_hashes = self.bktree.find(hash, similarity).collect::<Vec<_>>();
|
||||
if vector_with_found_similar_hashes.len() == 1 && vec_file_entry.len() == 1 {
|
||||
// This one picture doesn't have similar pictures, so there is no go
|
||||
continue;
|
||||
}
|
||||
for (hash, vec_file_entry) in this_time_check_hashes.iter() {
|
||||
let vector_with_found_similar_hashes = self
|
||||
.bktree
|
||||
.find(hash, similarity)
|
||||
.filter(|r| (r.0 == current_similarity) && !master_of_group.contains(r.1) && available_hashes.contains_key(r.1))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let mut vector_of_similar_images: Vec<FileEntry> = vec_file_entry
|
||||
.iter()
|
||||
.map(|fe| FileEntry {
|
||||
path: fe.path.clone(),
|
||||
size: fe.size,
|
||||
dimensions: fe.dimensions.clone(),
|
||||
modified_date: fe.modified_date,
|
||||
hash: fe.hash,
|
||||
similarity: Similarity::Similar(0),
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (similarity, similar_hash) in vector_with_found_similar_hashes.iter() {
|
||||
if *similarity == 0 && hash == *similar_hash {
|
||||
// This was already read before
|
||||
// Not found any hash with specific distance
|
||||
if vector_with_found_similar_hashes.is_empty() {
|
||||
continue;
|
||||
} else if hash == *similar_hash {
|
||||
panic!("I'm not sure if same hash can have distance > 0");
|
||||
}
|
||||
|
||||
if let Some(vec_file_entry) = non_cached_files_to_check.get(*similar_hash) {
|
||||
vector_of_similar_images.append(
|
||||
&mut (vec_file_entry
|
||||
// This one picture doesn't have similar pictures except self in similarity 0
|
||||
if current_similarity == 0 && vector_with_found_similar_hashes.len() == 1 {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Jeśli jeszcze nie dodał, to dodaje teraz grupę główną do już obrobionych
|
||||
if !master_of_group.contains(hash) {
|
||||
master_of_group.insert(*hash);
|
||||
collected_similar_images.insert(*hash, Vec::new());
|
||||
|
||||
let mut things: Vec<FileEntry> = vec_file_entry
|
||||
.iter()
|
||||
.map(|fe| FileEntry {
|
||||
path: fe.path.clone(),
|
||||
size: fe.size,
|
||||
dimensions: fe.dimensions.clone(),
|
||||
modified_date: fe.modified_date,
|
||||
hash: fe.hash,
|
||||
similarity: Similarity::Similar(0),
|
||||
})
|
||||
.collect();
|
||||
collected_similar_images.get_mut(hash).unwrap().append(&mut things);
|
||||
}
|
||||
|
||||
// Since we checked hash, we don't need to check it again
|
||||
if current_similarity != 0 {
|
||||
vector_with_found_similar_hashes.iter().for_each(|e| {
|
||||
let mut things: Vec<FileEntry> = available_hashes
|
||||
.get_mut(e.1)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|fe| FileEntry {
|
||||
path: fe.path.clone(),
|
||||
|
|
@ -516,20 +525,17 @@ impl SimilarImages {
|
|||
dimensions: fe.dimensions.clone(),
|
||||
modified_date: fe.modified_date,
|
||||
hash: [0; 8],
|
||||
similarity: Similarity::Similar(*similarity),
|
||||
similarity: Similarity::Similar(current_similarity),
|
||||
})
|
||||
.collect::<Vec<_>>()),
|
||||
);
|
||||
non_cached_files_to_check.remove(*similar_hash);
|
||||
.collect::<Vec<_>>();
|
||||
collected_similar_images.get_mut(hash).unwrap().append(&mut things);
|
||||
available_hashes.remove(e.1);
|
||||
});
|
||||
}
|
||||
}
|
||||
if vector_of_similar_images.len() > 1 {
|
||||
// Not sure why it may happens
|
||||
new_vector.push((*vector_of_similar_images).to_owned());
|
||||
}
|
||||
}
|
||||
|
||||
self.similar_vectors = new_vector;
|
||||
self.similar_vectors = collected_similar_images.values().cloned().collect();
|
||||
|
||||
Common::print_time(hash_map_modification, SystemTime::now(), "sort_images - selecting data from BtreeMap".to_string());
|
||||
|
||||
|
|
|
|||
|
|
@ -265,7 +265,7 @@ pub fn connect_button_search(
|
|||
|
||||
let minimal_file_size = entry_similar_images_minimal_size.text().as_str().parse::<u64>().unwrap_or(1024 * 16);
|
||||
|
||||
let similarity = similar_images::Similarity::Similar(scale_similarity.value() as u64);
|
||||
let similarity = similar_images::Similarity::Similar(scale_similarity.value() as u32);
|
||||
|
||||
let futures_sender_similar_images = futures_sender_similar_images.clone();
|
||||
// Find similar images
|
||||
|
|
|
|||
Loading…
Reference in New Issue