CodeMirror/legendary
1
0
Fork 0
mirror of https://github.com/derrod/legendary.git synced 2024-06-27 02:30:30 +12:00
Code Issues Projects Releases Wiki Activity

[downloader] Improve reordering optimizations

Browse source 
With some titles such as Metro Exodus there is
even more duplication across files. While this
change does not manage to reduce the limit to
below the default 1 GiB limit, it does bring
it down by about 512 MiB.
This commit is contained in:
derrod 2020-05-05 16:23:51 +02:00
parent 67859fb4ac
commit b1ba25e2e0
1 changed files with 18 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
legendary/downloader/manager.py
View file

@ -189,7 +189,7 @@ class DLManager(Process):
if processing_optimization: if processing_optimization:
# reorder the file manifest list to group files that share many chunks # reorder the file manifest list to group files that share many chunks
# 5 is mostly arbitrary but has shown in testing to be a good choice # 5 is mostly arbitrary but has shown in testing to be a good choice
min_overlap = 5 min_overlap = 4
# enumerate the file list to try and find a "partner" for # enumerate the file list to try and find a "partner" for
# each file that shares the most chunks with it. # each file that shares the most chunks with it.
partners = dict() partners = dict()
@ -197,13 +197,17 @@ class DLManager(Process):
for num, filename in enumerate(filenames[:int((len(filenames) + 1) / 2)]): for num, filename in enumerate(filenames[:int((len(filenames) + 1) / 2)]):
chunks = file_to_chunks[filename] chunks = file_to_chunks[filename]
max_overlap = min_overlap partnerlist = list()
for other_file in filenames[num + 1:]: for other_file in filenames[num + 1:]:
overlap = len(chunks & file_to_chunks[other_file]) overlap = len(chunks & file_to_chunks[other_file])
if overlap > max_overlap: if overlap > min_overlap:
partners[filename] = other_file partnerlist.append(other_file)
max_overlap = overlap
if not partnerlist:
continue
partners[filename] = partnerlist
# iterate over all the files again and this time around # iterate over all the files again and this time around
_fmlist = [] _fmlist = []
@ -214,13 +218,17 @@ class DLManager(Process):
_fmlist.append(fm) _fmlist.append(fm)
processed.add(fm.filename) processed.add(fm.filename)
# try to find the file's "partner" # try to find the file's "partner"
partner = partners.get(fm.filename, None) f_partners = partners.get(fm.filename, None)
if not partner or partner in processed: if not f_partners:
continue continue
# add each partner to list at this point
for partner in f_partners:
if partner in processed:
continue
partner_fm = manifest.file_manifest_list.get_file_by_path(partner) partner_fm = manifest.file_manifest_list.get_file_by_path(partner)
_fmlist.append(partner_fm) _fmlist.append(partner_fm)
processed.add(partner) processed.add(partner)
fmlist = _fmlist fmlist = _fmlist