Add in downloader parameters

bdfr/downloader.py

@@ -82,7 +82,7 @@ class RedditDownloader(RedditConnector):
                 logger.debug(f'Download filter removed {submission.id} file with URL {submission.url}')
                 continue
             try:
-                res.download()
+                res.download({'max_wait_time': self.args.max_wait_time})
             except errors.BulkDownloaderException as e:
                 logger.error(f'Failed to download resource {res.url} in submission {submission.id} '
                              f'with downloader {downloader_class.__name__}: {e}')

bdfr/resource.py

@@ -6,7 +6,6 @@ import logging
 import re
 import time
 import urllib.parse
-from collections import namedtuple
 from typing import Callable, Optional
 
 import _hashlib
@@ -33,8 +32,12 @@ class Resource:
     def retry_download(url: str) -> Callable:
         max_wait_time = 300
 
-        def http_download() -> Optional[bytes]:
+        def http_download(download_parameters: dict) -> Optional[bytes]:
             current_wait_time = 60
+            if 'max_wait_time' in download_parameters:
+                max_wait_time = download_parameters['max_wait_time']
+            else:
+                max_wait_time = 300
             while True:
                 try:
                     response = requests.get(url)
@@ -55,10 +58,12 @@ class Resource:
                         raise
         return http_download
 
-    def download(self):
+    def download(self, download_parameters: Optional[dict] = None):
+        if download_parameters is None:
+            download_parameters = {}
         if not self.content:
             try:
-                content = self.download_function()
+                content = self.download_function(download_parameters)
             except requests.exceptions.ConnectionError as e:
                 raise BulkDownloaderException(f'Could not download resource: {e}')
             except BulkDownloaderException:

bdfr/site_downloaders/youtube.py

@@ -40,7 +40,7 @@ class Youtube(BaseDownloader):
         ytdl_options['quiet'] = True
         ytdl_options['logger'] = yt_logger
 
-        def download() -> bytes:
+        def download(_: dict) -> bytes:
             with tempfile.TemporaryDirectory() as temp_dir:
                 download_path = Path(temp_dir).resolve()
                 ytdl_options['outtmpl'] = str(download_path) + '/' + 'test.%(ext)s'