CodeMirror/legendary
1
0
Fork 0
mirror of https://github.com/derrod/legendary.git synced 2024-06-24 17:21:08 +12:00
Code Issues Projects Releases Wiki Activity

[downloader.mp] Add exponential backoff to retries

Browse source 
Fixes #347
This commit is contained in:
derrod 2021-10-07 10:46:17 +02:00
parent a7d5e37f82
commit 2d366a7704
1 changed files with 7 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
legendary/downloader/mp/workers.py
View file

@ -19,7 +19,7 @@ from legendary.models.downloading import (
class DLWorker(Process):
def __init__(self, name, queue, out_queue, shm, max_retries=5,
def __init__(self, name, queue, out_queue, shm, max_retries=7,
logging_queue=None, dl_timeout=10):
super().__init__(name=name)
self.q = queue
@ -65,6 +65,12 @@ class DLWorker(Process):
try:
while tries < self.max_retries:
# retry once immediately, otherwise do exponential backoff
if tries > 1:
sleep_time = 2**(tries+1)
logger.debug(f'Sleeping {sleep_time} seconds before retrying...')
time.sleep(sleep_time)
# print('Downloading', job.url)
logger.debug(f'Downloading {job.url}')