From dbe8733fd44cb1b3055faa072c801e73e18d7865 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Tue, 27 Jul 2021 14:02:30 +1000 Subject: [PATCH] Refactor method to remove max wait time --- bdfr/resource.py | 5 ++++- bdfr/site_downloaders/direct.py | 2 +- bdfr/site_downloaders/erome.py | 2 +- bdfr/site_downloaders/gallery.py | 2 +- bdfr/site_downloaders/imgur.py | 2 +- bdfr/site_downloaders/redgifs.py | 2 +- tests/test_resource.py | 2 +- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/bdfr/resource.py b/bdfr/resource.py index 8f874ef..a1c90de 100644 --- a/bdfr/resource.py +++ b/bdfr/resource.py @@ -6,6 +6,7 @@ import logging import re import time import urllib.parse +from collections import namedtuple from typing import Callable, Optional import _hashlib @@ -29,7 +30,9 @@ class Resource: self.extension = self._determine_extension() @staticmethod - def retry_download(url: str, max_wait_time: int) -> Callable: + def retry_download(url: str) -> Callable: + max_wait_time = 300 + def http_download() -> Optional[bytes]: current_wait_time = 60 while True: diff --git a/bdfr/site_downloaders/direct.py b/bdfr/site_downloaders/direct.py index df1a469..833acae 100644 --- a/bdfr/site_downloaders/direct.py +++ b/bdfr/site_downloaders/direct.py @@ -14,4 +14,4 @@ class Direct(BaseDownloader): super().__init__(post) def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]: - return [Resource(self.post, self.post.url, Resource.retry_download(self.post.url, 300))] + return [Resource(self.post, self.post.url, Resource.retry_download(self.post.url))] diff --git a/bdfr/site_downloaders/erome.py b/bdfr/site_downloaders/erome.py index 69b9ae3..6130560 100644 --- a/bdfr/site_downloaders/erome.py +++ b/bdfr/site_downloaders/erome.py @@ -29,7 +29,7 @@ class Erome(BaseDownloader): for link in links: if not re.match(r'https?://.*', link): link = 'https://' + link - out.append(Resource(self.post, link, Resource.retry_download(link, 300))) + out.append(Resource(self.post, link, Resource.retry_download(link))) return out @staticmethod diff --git a/bdfr/site_downloaders/gallery.py b/bdfr/site_downloaders/gallery.py index c016d28..158e338 100644 --- a/bdfr/site_downloaders/gallery.py +++ b/bdfr/site_downloaders/gallery.py @@ -31,7 +31,7 @@ class Gallery(BaseDownloader): if not image_urls: raise SiteDownloaderError('No images found in Reddit gallery') - return [Resource(self.post, url, Resource.retry_download(url, 300)) for url in image_urls] + return [Resource(self.post, url, Resource.retry_download(url)) for url in image_urls] @ staticmethod def _get_links(id_dict: list[dict]) -> list[str]: diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py index 79a1115..f0b7012 100644 --- a/bdfr/site_downloaders/imgur.py +++ b/bdfr/site_downloaders/imgur.py @@ -33,7 +33,7 @@ class Imgur(BaseDownloader): def _compute_image_url(self, image: dict) -> Resource: image_url = 'https://i.imgur.com/' + image['hash'] + self._validate_extension(image['ext']) - return Resource(self.post, image_url, Resource.retry_download(image_url, 300)) + return Resource(self.post, image_url, Resource.retry_download(image_url)) @staticmethod def _get_data(link: str) -> dict: diff --git a/bdfr/site_downloaders/redgifs.py b/bdfr/site_downloaders/redgifs.py index d4989e7..a62fedb 100644 --- a/bdfr/site_downloaders/redgifs.py +++ b/bdfr/site_downloaders/redgifs.py @@ -18,7 +18,7 @@ class Redgifs(BaseDownloader): def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]: media_url = self._get_link(self.post.url) - return [Resource(self.post, media_url, Resource.retry_download(media_url, 300), '.mp4')] + return [Resource(self.post, media_url, Resource.retry_download(media_url), '.mp4')] @staticmethod def _get_link(url: str) -> str: diff --git a/tests/test_resource.py b/tests/test_resource.py index db9a6cc..f3bbc9a 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -31,6 +31,6 @@ def test_resource_get_extension(test_url: str, expected: str): ('https://www.iana.org/_img/2013.1/iana-logo-header.svg', '426b3ac01d3584c820f3b7f5985d6623'), )) def test_download_online_resource(test_url: str, expected_hash: str): - test_resource = Resource(MagicMock(), test_url, Resource.retry_download(test_url, 60)) + test_resource = Resource(MagicMock(), test_url, Resource.retry_download(test_url)) test_resource.download() assert test_resource.hash.hexdigest() == expected_hash