diff --git a/bdfr/downloader.py b/bdfr/downloader.py index 69aa818..70052b2 100644 --- a/bdfr/downloader.py +++ b/bdfr/downloader.py @@ -82,7 +82,7 @@ class RedditDownloader(RedditConnector): logger.debug(f'Download filter removed {submission.id} file with URL {submission.url}') continue try: - res.download() + res.download({'max_wait_time': self.args.max_wait_time}) except errors.BulkDownloaderException as e: logger.error(f'Failed to download resource {res.url} in submission {submission.id} ' f'with downloader {downloader_class.__name__}: {e}') diff --git a/bdfr/resource.py b/bdfr/resource.py index a1c90de..27ba84b 100644 --- a/bdfr/resource.py +++ b/bdfr/resource.py @@ -6,7 +6,6 @@ import logging import re import time import urllib.parse -from collections import namedtuple from typing import Callable, Optional import _hashlib @@ -33,8 +32,12 @@ class Resource: def retry_download(url: str) -> Callable: max_wait_time = 300 - def http_download() -> Optional[bytes]: + def http_download(download_parameters: dict) -> Optional[bytes]: current_wait_time = 60 + if 'max_wait_time' in download_parameters: + max_wait_time = download_parameters['max_wait_time'] + else: + max_wait_time = 300 while True: try: response = requests.get(url) @@ -55,10 +58,12 @@ class Resource: raise return http_download - def download(self): + def download(self, download_parameters: Optional[dict] = None): + if download_parameters is None: + download_parameters = {} if not self.content: try: - content = self.download_function() + content = self.download_function(download_parameters) except requests.exceptions.ConnectionError as e: raise BulkDownloaderException(f'Could not download resource: {e}') except BulkDownloaderException: diff --git a/bdfr/site_downloaders/youtube.py b/bdfr/site_downloaders/youtube.py index 126cb6a..a870c2e 100644 --- a/bdfr/site_downloaders/youtube.py +++ b/bdfr/site_downloaders/youtube.py @@ -40,7 +40,7 @@ class Youtube(BaseDownloader): ytdl_options['quiet'] = True ytdl_options['logger'] = yt_logger - def download() -> bytes: + def download(_: dict) -> bytes: with tempfile.TemporaryDirectory() as temp_dir: download_path = Path(temp_dir).resolve() ytdl_options['outtmpl'] = str(download_path) + '/' + 'test.%(ext)s'