2021-02-11 12:09:49 +13:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
# coding=utf-8
|
|
|
|
|
|
|
|
import hashlib
|
2021-03-11 16:20:39 +13:00
|
|
|
import logging
|
2021-02-11 12:09:49 +13:00
|
|
|
import re
|
2021-02-26 21:56:05 +13:00
|
|
|
import time
|
|
|
|
from typing import Optional
|
2021-02-11 12:09:49 +13:00
|
|
|
|
2021-02-26 21:56:05 +13:00
|
|
|
import _hashlib
|
|
|
|
import requests
|
2021-02-11 12:09:49 +13:00
|
|
|
from praw.models import Submission
|
|
|
|
|
2021-04-12 19:58:32 +12:00
|
|
|
from bdfr.exceptions import BulkDownloaderException
|
2021-02-26 21:56:05 +13:00
|
|
|
|
2021-03-11 16:20:39 +13:00
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
2021-02-11 12:09:49 +13:00
|
|
|
|
|
|
|
class Resource:
|
2021-02-26 21:56:05 +13:00
|
|
|
def __init__(self, source_submission: Submission, url: str, extension: str = None):
|
2021-02-11 12:09:49 +13:00
|
|
|
self.source_submission = source_submission
|
2021-02-26 21:56:05 +13:00
|
|
|
self.content: Optional[bytes] = None
|
2021-02-11 12:09:49 +13:00
|
|
|
self.url = url
|
2021-02-26 21:56:05 +13:00
|
|
|
self.hash: Optional[_hashlib.HASH] = None
|
|
|
|
self.extension = extension
|
|
|
|
if not self.extension:
|
|
|
|
self.extension = self._determine_extension()
|
2021-02-11 12:09:49 +13:00
|
|
|
|
|
|
|
@staticmethod
|
2021-04-12 18:47:04 +12:00
|
|
|
def retry_download(url: str, max_wait_time: int) -> Optional[bytes]:
|
|
|
|
wait_time = 60
|
2021-02-26 21:56:05 +13:00
|
|
|
try:
|
|
|
|
response = requests.get(url)
|
2021-04-12 20:59:43 +12:00
|
|
|
if re.match(r'^2\d{2}', str(response.status_code)) and response.content:
|
2021-02-26 21:56:05 +13:00
|
|
|
return response.content
|
2021-04-12 20:21:04 +12:00
|
|
|
elif response.status_code in (408, 429):
|
|
|
|
raise requests.exceptions.ConnectionError(f'Response code {response.status_code}')
|
|
|
|
else:
|
2021-04-08 13:08:12 +12:00
|
|
|
raise BulkDownloaderException(
|
|
|
|
f'Unrecoverable error requesting resource: HTTP Code {response.status_code}')
|
2021-04-05 17:11:17 +12:00
|
|
|
except requests.exceptions.ConnectionError as e:
|
2021-04-08 15:00:58 +12:00
|
|
|
logger.warning(f'Error occured downloading from {url}, waiting {wait_time} seconds: {e}')
|
2021-02-26 21:56:05 +13:00
|
|
|
time.sleep(wait_time)
|
2021-04-12 18:47:04 +12:00
|
|
|
if wait_time < max_wait_time:
|
|
|
|
return Resource.retry_download(url, max_wait_time)
|
2021-02-26 21:56:05 +13:00
|
|
|
else:
|
2021-03-11 16:20:39 +13:00
|
|
|
logger.error(f'Max wait time exceeded for resource at url {url}')
|
2021-04-05 17:11:17 +12:00
|
|
|
raise
|
2021-02-26 21:56:05 +13:00
|
|
|
|
2021-04-12 18:47:04 +12:00
|
|
|
def download(self, max_wait_time: int):
|
2021-02-26 21:56:05 +13:00
|
|
|
if not self.content:
|
2021-04-05 17:11:17 +12:00
|
|
|
try:
|
2021-04-12 18:47:04 +12:00
|
|
|
content = self.retry_download(self.url, max_wait_time)
|
2021-04-05 17:11:17 +12:00
|
|
|
except requests.exceptions.ConnectionError as e:
|
|
|
|
raise BulkDownloaderException(f'Could not download resource: {e}')
|
|
|
|
except BulkDownloaderException:
|
|
|
|
raise
|
2021-02-26 21:56:05 +13:00
|
|
|
if content:
|
|
|
|
self.content = content
|
2021-04-05 17:11:17 +12:00
|
|
|
if not self.hash and self.content:
|
|
|
|
self.create_hash()
|
2021-02-26 21:56:05 +13:00
|
|
|
|
2021-02-28 12:40:42 +13:00
|
|
|
def create_hash(self):
|
|
|
|
self.hash = hashlib.md5(self.content)
|
|
|
|
|
2021-04-13 15:17:40 +12:00
|
|
|
def _determine_extension(self) -> Optional[str]:
|
2021-03-10 17:07:59 +13:00
|
|
|
extension_pattern = re.compile(r'.*(\..{3,5})(?:\?.*)?$')
|
2021-02-26 21:56:05 +13:00
|
|
|
match = re.search(extension_pattern, self.url)
|
|
|
|
if match:
|
|
|
|
return match.group(1)
|