Catch requests errors in site downloaders
This commit is contained in:
parent
d960bc0b7b
commit
6767777944
1 changed files with 6 additions and 2 deletions
|
@ -8,7 +8,7 @@ from typing import Optional
|
|||
import requests
|
||||
from praw.models import Submission
|
||||
|
||||
from bdfr.exceptions import ResourceNotFound
|
||||
from bdfr.exceptions import ResourceNotFound, SiteDownloaderError
|
||||
from bdfr.resource import Resource
|
||||
from bdfr.site_authenticator import SiteAuthenticator
|
||||
|
||||
|
@ -27,7 +27,11 @@ class BaseDownloader(ABC):
|
|||
|
||||
@staticmethod
|
||||
def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
|
||||
try:
|
||||
res = requests.get(url, cookies=cookies, headers=headers)
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.exception(e)
|
||||
raise SiteDownloaderError(f'Failed to get page {url}')
|
||||
if res.status_code != 200:
|
||||
raise ResourceNotFound(f'Server responded with {res.status_code} to {url}')
|
||||
return res
|
||||
|
|
Loading…
Reference in a new issue