Add in downloader parameters
This commit is contained in:
parent
dbe8733fd4
commit
7bca303b1b
3 changed files with 11 additions and 6 deletions
|
@ -82,7 +82,7 @@ class RedditDownloader(RedditConnector):
|
||||||
logger.debug(f'Download filter removed {submission.id} file with URL {submission.url}')
|
logger.debug(f'Download filter removed {submission.id} file with URL {submission.url}')
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
res.download()
|
res.download({'max_wait_time': self.args.max_wait_time})
|
||||||
except errors.BulkDownloaderException as e:
|
except errors.BulkDownloaderException as e:
|
||||||
logger.error(f'Failed to download resource {res.url} in submission {submission.id} '
|
logger.error(f'Failed to download resource {res.url} in submission {submission.id} '
|
||||||
f'with downloader {downloader_class.__name__}: {e}')
|
f'with downloader {downloader_class.__name__}: {e}')
|
||||||
|
|
|
@ -6,7 +6,6 @@ import logging
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
from collections import namedtuple
|
|
||||||
from typing import Callable, Optional
|
from typing import Callable, Optional
|
||||||
|
|
||||||
import _hashlib
|
import _hashlib
|
||||||
|
@ -33,8 +32,12 @@ class Resource:
|
||||||
def retry_download(url: str) -> Callable:
|
def retry_download(url: str) -> Callable:
|
||||||
max_wait_time = 300
|
max_wait_time = 300
|
||||||
|
|
||||||
def http_download() -> Optional[bytes]:
|
def http_download(download_parameters: dict) -> Optional[bytes]:
|
||||||
current_wait_time = 60
|
current_wait_time = 60
|
||||||
|
if 'max_wait_time' in download_parameters:
|
||||||
|
max_wait_time = download_parameters['max_wait_time']
|
||||||
|
else:
|
||||||
|
max_wait_time = 300
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
response = requests.get(url)
|
response = requests.get(url)
|
||||||
|
@ -55,10 +58,12 @@ class Resource:
|
||||||
raise
|
raise
|
||||||
return http_download
|
return http_download
|
||||||
|
|
||||||
def download(self):
|
def download(self, download_parameters: Optional[dict] = None):
|
||||||
|
if download_parameters is None:
|
||||||
|
download_parameters = {}
|
||||||
if not self.content:
|
if not self.content:
|
||||||
try:
|
try:
|
||||||
content = self.download_function()
|
content = self.download_function(download_parameters)
|
||||||
except requests.exceptions.ConnectionError as e:
|
except requests.exceptions.ConnectionError as e:
|
||||||
raise BulkDownloaderException(f'Could not download resource: {e}')
|
raise BulkDownloaderException(f'Could not download resource: {e}')
|
||||||
except BulkDownloaderException:
|
except BulkDownloaderException:
|
||||||
|
|
|
@ -40,7 +40,7 @@ class Youtube(BaseDownloader):
|
||||||
ytdl_options['quiet'] = True
|
ytdl_options['quiet'] = True
|
||||||
ytdl_options['logger'] = yt_logger
|
ytdl_options['logger'] = yt_logger
|
||||||
|
|
||||||
def download() -> bytes:
|
def download(_: dict) -> bytes:
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
download_path = Path(temp_dir).resolve()
|
download_path = Path(temp_dir).resolve()
|
||||||
ytdl_options['outtmpl'] = str(download_path) + '/' + 'test.%(ext)s'
|
ytdl_options['outtmpl'] = str(download_path) + '/' + 'test.%(ext)s'
|
||||||
|
|
Loading…
Reference in a new issue