CodeMirror/czkawka
1
0
Fork 0
mirror of synced 2024-05-19 11:53:43 +12:00
Code Issues Projects Releases Wiki Activity

Simplify music checking

Browse source
This commit is contained in:
Rafał Mikrut 2023-05-02 11:59:26 +02:00
parent 92500792df
commit ec0d56776c
2 changed files with 360 additions and 371 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

282
czkawka_core/src/duplicate.rs
View file

@ -11,7 +11,6 @@ use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::SystemTime;
use std::{fs, mem};
@ -644,36 +643,15 @@ impl DuplicateFinder {
}
}
fn prehashing(
&mut self,
stop_receiver: Option<&Receiver<()>>,
progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>,
pre_checked_map: &mut BTreeMap<u64, Vec<FileEntry>>,
) -> Option<()> {
let start_time: SystemTime = SystemTime::now();
let check_type = self.hash_type;
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
1,
2,
self.files_with_identical_size.values().map(Vec::len).sum(),
self.check_method,
);
let loaded_hash_map;
let mut records_already_cached: BTreeMap<u64, Vec<FileEntry>> = Default::default();
let mut non_cached_files_to_check: BTreeMap<u64, Vec<FileEntry>> = Default::default();
fn prehash_load_cache_at_start(&mut self) -> (BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>) {
// Cache algorithm
// - Load data from cache
// - Convert from BT<u64,Vec<FileEntry>> to BT<String,FileEntry>
// - Save to proper values
let loaded_hash_map;
let mut records_already_cached: BTreeMap<u64, Vec<FileEntry>> = Default::default();
let mut non_cached_files_to_check: BTreeMap<u64, Vec<FileEntry>> = Default::default();
if self.use_prehash_cache {
loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache, &self.hash_type, true) {
Some(t) => t,
@ -707,6 +685,59 @@ impl DuplicateFinder {
loaded_hash_map = Default::default();
mem::swap(&mut self.files_with_identical_size, &mut non_cached_files_to_check);
}
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
fn prehash_save_cache_at_exit(&mut self, loaded_hash_map: BTreeMap<u64, Vec<FileEntry>>, pre_hash_results: &Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)>) {
if self.use_prehash_cache {
// All results = records already cached + computed results
let mut save_cache_to_hashmap: BTreeMap<String, FileEntry> = Default::default();
for (size, vec_file_entry) in loaded_hash_map {
if size >= self.minimal_prehash_cache_file_size {
for file_entry in vec_file_entry {
save_cache_to_hashmap.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone());
}
}
}
for (size, hash_map, _errors) in pre_hash_results {
if *size >= self.minimal_prehash_cache_file_size {
for vec_file_entry in hash_map.values() {
for file_entry in vec_file_entry {
save_cache_to_hashmap.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone());
}
}
}
}
save_hashes_to_file(&save_cache_to_hashmap, &mut self.text_messages, &self.hash_type, true, self.minimal_prehash_cache_file_size);
}
}
fn prehashing(
&mut self,
stop_receiver: Option<&Receiver<()>>,
progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>,
pre_checked_map: &mut BTreeMap<u64, Vec<FileEntry>>,
) -> Option<()> {
let start_time: SystemTime = SystemTime::now();
let check_type = self.hash_type;
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
let progress_thread_run = Arc::new(AtomicBool::new(true));
let atomic_counter = Arc::new(AtomicUsize::new(0));
let progress_thread_handle = prepare_thread_handler_common(
progress_sender,
&progress_thread_run,
&atomic_counter,
1,
2,
self.files_with_identical_size.values().map(Vec::len).sum(),
self.check_method,
);
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.prehash_load_cache_at_start();
#[allow(clippy::type_complexity)]
let pre_hash_results: Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)> = non_cached_files_to_check
@ -756,41 +787,107 @@ impl DuplicateFinder {
}
}
if self.use_prehash_cache {
// All results = records already cached + computed results
let mut save_cache_to_hashmap: BTreeMap<String, FileEntry> = Default::default();
for (size, vec_file_entry) in loaded_hash_map {
if size >= self.minimal_prehash_cache_file_size {
for file_entry in vec_file_entry {
save_cache_to_hashmap.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone());
}
}
}
for (size, hash_map, _errors) in &pre_hash_results {
if *size >= self.minimal_prehash_cache_file_size {
for vec_file_entry in hash_map.values() {
for file_entry in vec_file_entry {
save_cache_to_hashmap.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone());
}
}
}
}
save_hashes_to_file(&save_cache_to_hashmap, &mut self.text_messages, &self.hash_type, true, self.minimal_prehash_cache_file_size);
}
self.prehash_save_cache_at_exit(loaded_hash_map, &pre_hash_results);
Common::print_time(start_time, SystemTime::now(), "check_files_hash - prehash");
Some(())
}
fn full_hashing_load_cache_at_start(
&mut self,
mut pre_checked_map: BTreeMap<u64, Vec<FileEntry>>,
) -> (BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>, BTreeMap<u64, Vec<FileEntry>>) {
let loaded_hash_map;
let mut records_already_cached: BTreeMap<u64, Vec<FileEntry>> = Default::default();
let mut non_cached_files_to_check: BTreeMap<u64, Vec<FileEntry>> = Default::default();
if self.use_cache {
loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache, &self.hash_type, false) {
Some(t) => t,
None => Default::default(),
};
for (size, vec_file_entry) in pre_checked_map {
#[allow(clippy::collapsible_if)]
if !loaded_hash_map.contains_key(&size) {
// If loaded data doesn't contains current info
non_cached_files_to_check.insert(size, vec_file_entry);
} else {
let loaded_vec_file_entry = loaded_hash_map.get(&size).unwrap();
for file_entry in vec_file_entry {
let mut found: bool = false;
for loaded_file_entry in loaded_vec_file_entry {
if file_entry.path == loaded_file_entry.path && file_entry.modified_date == loaded_file_entry.modified_date {
records_already_cached.entry(file_entry.size).or_insert_with(Vec::new).push(loaded_file_entry.clone());
found = true;
break;
}
}
if !found {
non_cached_files_to_check.entry(file_entry.size).or_insert_with(Vec::new).push(file_entry);
}
}
}
}
} else {
loaded_hash_map = Default::default();
mem::swap(&mut pre_checked_map, &mut non_cached_files_to_check);
}
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
fn full_hashing_save_cache_at_exit(
&mut self,
records_already_cached: BTreeMap<u64, Vec<FileEntry>>,
full_hash_results: &mut Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)>,
loaded_hash_map: BTreeMap<u64, Vec<FileEntry>>,
) {
if !self.use_cache {
return;
}
'main: for (size, vec_file_entry) in records_already_cached {
// Check if size already exists, if exists we must to change it outside because cannot have mut and non mut reference to full_hash_results
for (full_size, full_hashmap, _errors) in &mut (*full_hash_results) {
if size == *full_size {
for file_entry in vec_file_entry {
full_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry);
}
continue 'main;
}
}
// Size doesn't exists add results to files
let mut temp_hashmap: BTreeMap<String, Vec<FileEntry>> = Default::default();
for file_entry in vec_file_entry {
temp_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry);
}
full_hash_results.push((size, temp_hashmap, Vec::new()));
}
// Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = Default::default();
for (_size, vec_file_entry) in loaded_hash_map {
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
}
for (_size, hashmap, _errors) in full_hash_results {
for vec_file_entry in hashmap.values() {
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone());
}
}
}
save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type, false, self.minimal_cache_file_size);
}
fn full_hashing(
&mut self,
stop_receiver: Option<&Receiver<()>>,
progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>,
mut pre_checked_map: BTreeMap<u64, Vec<FileEntry>>,
pre_checked_map: BTreeMap<u64, Vec<FileEntry>>,
) -> Option<()> {
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
@ -812,50 +909,9 @@ impl DuplicateFinder {
///////////////////////////////////////////////////////////////////////////// HASHING START
{
#[allow(clippy::type_complexity)]
let mut full_hash_results: Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)>;
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.full_hashing_load_cache_at_start(pre_checked_map);
let loaded_hash_map;
let mut records_already_cached: BTreeMap<u64, Vec<FileEntry>> = Default::default();
let mut non_cached_files_to_check: BTreeMap<u64, Vec<FileEntry>> = Default::default();
if self.use_cache {
loaded_hash_map = match load_hashes_from_file(&mut self.text_messages, self.delete_outdated_cache, &self.hash_type, false) {
Some(t) => t,
None => Default::default(),
};
for (size, vec_file_entry) in pre_checked_map {
#[allow(clippy::collapsible_if)]
if !loaded_hash_map.contains_key(&size) {
// If loaded data doesn't contains current info
non_cached_files_to_check.insert(size, vec_file_entry);
} else {
let loaded_vec_file_entry = loaded_hash_map.get(&size).unwrap();
for file_entry in vec_file_entry {
let mut found: bool = false;
for loaded_file_entry in loaded_vec_file_entry {
if file_entry.path == loaded_file_entry.path && file_entry.modified_date == loaded_file_entry.modified_date {
records_already_cached.entry(file_entry.size).or_insert_with(Vec::new).push(loaded_file_entry.clone());
found = true;
break;
}
}
if !found {
non_cached_files_to_check.entry(file_entry.size).or_insert_with(Vec::new).push(file_entry);
}
}
}
}
} else {
loaded_hash_map = Default::default();
mem::swap(&mut pre_checked_map, &mut non_cached_files_to_check);
}
full_hash_results = non_cached_files_to_check
let mut full_hash_results: Vec<(u64, BTreeMap<String, Vec<FileEntry>>, Vec<String>)> = non_cached_files_to_check
.into_par_iter()
.map(|(size, vec_file_entry)| {
let mut hashmap_with_hash: BTreeMap<String, Vec<FileEntry>> = Default::default();
@ -882,41 +938,7 @@ impl DuplicateFinder {
.while_some()
.collect();
if self.use_cache {
'main: for (size, vec_file_entry) in records_already_cached {
// Check if size already exists, if exists we must to change it outside because cannot have mut and non mut reference to full_hash_results
for (full_size, full_hashmap, _errors) in &mut full_hash_results {
if size == *full_size {
for file_entry in vec_file_entry {
full_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry);
}
continue 'main;
}
}
// Size doesn't exists add results to files
let mut temp_hashmap: BTreeMap<String, Vec<FileEntry>> = Default::default();
for file_entry in vec_file_entry {
temp_hashmap.entry(file_entry.hash.clone()).or_insert_with(Vec::new).push(file_entry);
}
full_hash_results.push((size, temp_hashmap, Vec::new()));
}
// Must save all results to file, old loaded from file with all currently counted results
let mut all_results: BTreeMap<String, FileEntry> = Default::default();
for (_size, vec_file_entry) in loaded_hash_map {
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
}
for (_size, hashmap, _errors) in &full_hash_results {
for vec_file_entry in hashmap.values() {
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry.clone());
}
}
}
save_hashes_to_file(&all_results, &mut self.text_messages, &self.hash_type, false, self.minimal_cache_file_size);
}
self.full_hashing_save_cache_at_exit(records_already_cached, &mut full_hash_results, loaded_hash_map);
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);

449
czkawka_core/src/same_music.rs
View file

@ -5,7 +5,6 @@ use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::SystemTime;
use std::{mem, panic};
@ -323,9 +322,7 @@ impl SameMusic {
}
}
fn read_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>) -> bool {
let start_time: SystemTime = SystemTime::now();
fn read_tags_load_cache(&mut self) -> (HashMap<String, MusicEntry>, HashMap<String, MusicEntry>, HashMap<String, MusicEntry>) {
let loaded_hash_map;
let mut records_already_cached: HashMap<String, MusicEntry> = Default::default();
@ -354,6 +351,26 @@ impl SameMusic {
loaded_hash_map = Default::default();
mem::swap(&mut self.music_to_check, &mut non_cached_files_to_check);
}
(loaded_hash_map, records_already_cached, non_cached_files_to_check)
}
fn read_tags_save_cache(&mut self, vec_file_entry: Vec<MusicEntry>, loaded_hash_map: HashMap<String, MusicEntry>) {
if !self.use_cache {
return;
}
// Must save all results to file, old loaded from file with all currently counted results
let mut all_results: HashMap<String, MusicEntry> = loaded_hash_map;
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
save_cache_to_file(&all_results, &mut self.text_messages, self.save_also_as_json);
}
fn read_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>) -> bool {
let start_time: SystemTime = SystemTime::now();
let (loaded_hash_map, records_already_cached, non_cached_files_to_check) = self.read_tags_load_cache();
let check_was_stopped = AtomicBool::new(false); // Used for breaking from GUI and ending check thread
@ -372,103 +389,13 @@ impl SameMusic {
// Clean for duplicate files
let mut vec_file_entry = non_cached_files_to_check
.into_par_iter()
.map(|(path, mut music_entry)| {
.map(|(path, music_entry)| {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
check_was_stopped.store(true, Ordering::Relaxed);
return None;
}
let Ok(mut file) = File::open(&path) else{return Some(None)};
let result = panic::catch_unwind(move || {
match read_from(&mut file) {
Ok(t) => Some(t),
Err(_inspected) => {
// println!("Failed to open {}", path);
None
}
}
});
let tagged_file = if let Ok(t) = result {
match t {
Some(r) => r,
None => {
return Some(Some(music_entry));
}
}
} else {
let message = create_crash_message("Lofty", &path, "https://github.com/image-rs/image/issues");
println!("{message}");
return Some(None);
};
let properties = tagged_file.properties();
let mut track_title = String::new();
let mut track_artist = String::new();
let mut year = String::new();
let mut genre = String::new();
let bitrate = properties.audio_bitrate().unwrap_or(0);
let mut length = properties.duration().as_millis().to_string();
if let Some(tag) = tagged_file.primary_tag() {
track_title = tag.get_string(&ItemKey::TrackTitle).unwrap_or("").to_string();
track_artist = tag.get_string(&ItemKey::TrackArtist).unwrap_or("").to_string();
year = tag.get_string(&ItemKey::Year).unwrap_or("").to_string();
genre = tag.get_string(&ItemKey::Genre).unwrap_or("").to_string();
}
for tag in tagged_file.tags() {
if track_title.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::TrackTitle) {
track_title = tag_value.to_string();
}
}
if track_artist.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::TrackArtist) {
track_artist = tag_value.to_string();
}
}
if year.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::Year) {
year = tag_value.to_string();
}
}
if genre.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::Genre) {
genre = tag_value.to_string();
}
}
// println!("{:?}", tag.items());
}
if let Ok(old_length_number) = length.parse::<u32>() {
let length_number = old_length_number / 60;
let minutes = length_number / 1000;
let seconds = (length_number % 1000) * 6 / 100;
if minutes != 0 || seconds != 0 {
length = format!("{minutes}:{seconds:02}");
} else if old_length_number > 0 {
// That means, that audio have length smaller that second, but length is properly read
length = "0:01".to_string();
} else {
length = String::new();
}
} else {
length = String::new();
}
music_entry.track_title = track_title;
music_entry.track_artist = track_artist;
music_entry.year = year;
music_entry.length = length;
music_entry.genre = genre;
music_entry.bitrate = bitrate;
Some(Some(music_entry))
Some(self.read_single_file_tag(&path, music_entry))
})
.while_some()
.filter(Option::is_some)
@ -484,15 +411,7 @@ impl SameMusic {
self.music_entries = vec_file_entry.clone();
if self.use_cache {
// Must save all results to file, old loaded from file with all currently counted results
let mut all_results: HashMap<String, MusicEntry> = loaded_hash_map;
for file_entry in vec_file_entry {
all_results.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
}
save_cache_to_file(&all_results, &mut self.text_messages, self.save_also_as_json);
}
self.read_tags_save_cache(vec_file_entry, loaded_hash_map);
// Break if stop was clicked after saving to cache
if check_was_stopped.load(Ordering::Relaxed) {
@ -503,6 +422,99 @@ impl SameMusic {
true
}
fn read_single_file_tag(&self, path: &str, mut music_entry: MusicEntry) -> Option<MusicEntry> {
let Ok(mut file) = File::open(path) else{return None};
let result = panic::catch_unwind(move || {
match read_from(&mut file) {
Ok(t) => Some(t),
Err(_inspected) => {
// println!("Failed to open {}", path);
None
}
}
});
let tagged_file = if let Ok(t) = result {
match t {
Some(r) => r,
None => {
return Some(music_entry);
}
}
} else {
let message = create_crash_message("Lofty", path, "https://github.com/image-rs/image/issues");
println!("{message}");
return None;
};
let properties = tagged_file.properties();
let mut track_title = String::new();
let mut track_artist = String::new();
let mut year = String::new();
let mut genre = String::new();
let bitrate = properties.audio_bitrate().unwrap_or(0);
let mut length = properties.duration().as_millis().to_string();
if let Some(tag) = tagged_file.primary_tag() {
track_title = tag.get_string(&ItemKey::TrackTitle).unwrap_or("").to_string();
track_artist = tag.get_string(&ItemKey::TrackArtist).unwrap_or("").to_string();
year = tag.get_string(&ItemKey::Year).unwrap_or("").to_string();
genre = tag.get_string(&ItemKey::Genre).unwrap_or("").to_string();
}
for tag in tagged_file.tags() {
if track_title.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::TrackTitle) {
track_title = tag_value.to_string();
}
}
if track_artist.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::TrackArtist) {
track_artist = tag_value.to_string();
}
}
if year.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::Year) {
year = tag_value.to_string();
}
}
if genre.is_empty() {
if let Some(tag_value) = tag.get_string(&ItemKey::Genre) {
genre = tag_value.to_string();
}
}
// println!("{:?}", tag.items());
}
if let Ok(old_length_number) = length.parse::<u32>() {
let length_number = old_length_number / 60;
let minutes = length_number / 1000;
let seconds = (length_number % 1000) * 6 / 100;
if minutes != 0 || seconds != 0 {
length = format!("{minutes}:{seconds:02}");
} else if old_length_number > 0 {
// That means, that audio have length smaller that second, but length is properly read
length = "0:01".to_string();
} else {
length = String::new();
}
} else {
length = String::new();
}
music_entry.track_title = track_title;
music_entry.track_artist = track_artist;
music_entry.year = year;
music_entry.length = length;
music_entry.genre = genre;
music_entry.bitrate = bitrate;
Some(music_entry)
}
fn check_for_duplicate_tags(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&futures::channel::mpsc::UnboundedSender<ProgressData>>) -> bool {
assert_ne!(MusicSimilarity::NONE, self.music_similarity, "This can't be none");
let start_time: SystemTime = SystemTime::now();
@ -523,133 +535,53 @@ impl SameMusic {
let mut new_duplicates: Vec<Vec<MusicEntry>> = Vec::new();
if (self.music_similarity & MusicSimilarity::TRACK_TITLE) == MusicSimilarity::TRACK_TITLE {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
for file_entry in vec_file_entry {
let mut thing = file_entry.track_title.trim().to_lowercase();
if self.approximate_comparison {
get_approximate_conversion(&mut thing);
}
if !thing.is_empty() {
hash_map.entry(thing.clone()).or_insert_with(Vec::new).push(file_entry);
}
}
for (_title, vec_file_entry) in hash_map {
if vec_file_entry.len() > 1 {
new_duplicates.push(vec_file_entry);
}
}
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
old_duplicates = new_duplicates;
new_duplicates = Vec::new();
old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.track_title, self.approximate_comparison);
}
if (self.music_similarity & MusicSimilarity::TRACK_ARTIST) == MusicSimilarity::TRACK_ARTIST {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
for file_entry in vec_file_entry {
let mut thing = file_entry.track_artist.trim().to_lowercase();
if self.approximate_comparison {
get_approximate_conversion(&mut thing);
}
if !thing.is_empty() {
hash_map.entry(thing.clone()).or_insert_with(Vec::new).push(file_entry);
}
}
for (_title, vec_file_entry) in hash_map {
if vec_file_entry.len() > 1 {
new_duplicates.push(vec_file_entry);
}
}
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
old_duplicates = new_duplicates;
new_duplicates = Vec::new();
old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.track_artist, self.approximate_comparison);
}
if (self.music_similarity & MusicSimilarity::YEAR) == MusicSimilarity::YEAR {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
for file_entry in vec_file_entry {
let thing = file_entry.year.trim().to_lowercase();
if !thing.is_empty() {
hash_map.entry(thing.clone()).or_insert_with(Vec::new).push(file_entry);
}
}
for (_title, vec_file_entry) in hash_map {
if vec_file_entry.len() > 1 {
new_duplicates.push(vec_file_entry);
}
}
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
old_duplicates = new_duplicates;
new_duplicates = Vec::new();
old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.year, false);
}
if (self.music_similarity & MusicSimilarity::LENGTH) == MusicSimilarity::LENGTH {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
for file_entry in vec_file_entry {
let thing = file_entry.length.trim().to_lowercase();
if !thing.is_empty() {
hash_map.entry(thing.clone()).or_insert_with(Vec::new).push(file_entry);
}
}
for (_title, vec_file_entry) in hash_map {
if vec_file_entry.len() > 1 {
new_duplicates.push(vec_file_entry);
}
}
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
old_duplicates = new_duplicates;
new_duplicates = Vec::new();
old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.length, false);
}
if (self.music_similarity & MusicSimilarity::GENRE) == MusicSimilarity::GENRE {
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
for file_entry in vec_file_entry {
let thing = file_entry.genre.trim().to_lowercase();
if !thing.is_empty() {
hash_map.entry(thing).or_insert_with(Vec::new).push(file_entry);
}
}
for (_title, vec_file_entry) in hash_map {
if vec_file_entry.len() > 1 {
new_duplicates.push(vec_file_entry);
}
}
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
old_duplicates = new_duplicates;
new_duplicates = Vec::new();
old_duplicates = self.check_music_item(old_duplicates, &atomic_counter, |fe| &fe.genre, false);
}
if (self.music_similarity & MusicSimilarity::BITRATE) == MusicSimilarity::BITRATE {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
}
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
for file_entry in vec_file_entry {
if file_entry.bitrate != 0 {
@ -666,38 +598,13 @@ impl SameMusic {
}
}
old_duplicates = new_duplicates;
// new_duplicates = Vec::new();
}
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
self.duplicated_music_entries = old_duplicates;
if self.use_reference_folders {
let mut similar_vector = Default::default();
mem::swap(&mut self.duplicated_music_entries, &mut similar_vector);
let reference_directories = self.directories.reference_directories.clone();
self.duplicated_music_entries_referenced = similar_vector
.into_iter()
.filter_map(|vec_file_entry| {
let mut files_from_referenced_folders = Vec::new();
let mut normal_files = Vec::new();
for file_entry in vec_file_entry {
if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) {
files_from_referenced_folders.push(file_entry);
} else {
normal_files.push(file_entry);
}
}
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
} else {
Some((files_from_referenced_folders.pop().unwrap(), normal_files))
}
})
.collect::<Vec<(MusicEntry, Vec<MusicEntry>)>>();
}
self.filter_reference_folders();
if self.use_reference_folders {
for (_fe, vector) in &self.duplicated_music_entries_referenced {
@ -719,6 +626,66 @@ impl SameMusic {
true
}
fn check_music_item(
&self,
old_duplicates: Vec<Vec<MusicEntry>>,
atomic_counter: &Arc<AtomicUsize>,
get_item: fn(&MusicEntry) -> &str,
approximate_comparison: bool,
) -> Vec<Vec<MusicEntry>> {
let mut new_duplicates: Vec<_> = Default::default();
for vec_file_entry in old_duplicates {
atomic_counter.fetch_add(1, Ordering::Relaxed);
let mut hash_map: BTreeMap<String, Vec<MusicEntry>> = Default::default();
for file_entry in vec_file_entry {
let mut thing = get_item(&file_entry).trim().to_lowercase();
if approximate_comparison {
get_approximate_conversion(&mut thing);
}
if !thing.is_empty() {
hash_map.entry(thing).or_insert_with(Vec::new).push(file_entry);
}
}
for (_title, vec_file_entry) in hash_map {
if vec_file_entry.len() > 1 {
new_duplicates.push(vec_file_entry);
}
}
}
new_duplicates
}
fn filter_reference_folders(&mut self) {
if !self.use_reference_folders {
return;
}
let mut similar_vector = Default::default();
mem::swap(&mut self.duplicated_music_entries, &mut similar_vector);
let reference_directories = self.directories.reference_directories.clone();
self.duplicated_music_entries_referenced = similar_vector
.into_iter()
.filter_map(|vec_file_entry| {
let mut files_from_referenced_folders = Vec::new();
let mut normal_files = Vec::new();
for file_entry in vec_file_entry {
if reference_directories.iter().any(|e| file_entry.path.starts_with(e)) {
files_from_referenced_folders.push(file_entry);
} else {
normal_files.push(file_entry);
}
}
if files_from_referenced_folders.is_empty() || normal_files.is_empty() {
None
} else {
Some((files_from_referenced_folders.pop().unwrap(), normal_files))
}
})
.collect::<Vec<(MusicEntry, Vec<MusicEntry>)>>();
}
pub fn set_minimal_file_size(&mut self, minimal_file_size: u64) {
self.minimal_file_size = match minimal_file_size {
0 => 1,