Catch requests errors in site downloaders
This commit is contained in:
parent
d960bc0b7b
commit
6767777944
1 changed files with 6 additions and 2 deletions
|
@ -8,7 +8,7 @@ from typing import Optional
|
||||||
import requests
|
import requests
|
||||||
from praw.models import Submission
|
from praw.models import Submission
|
||||||
|
|
||||||
from bdfr.exceptions import ResourceNotFound
|
from bdfr.exceptions import ResourceNotFound, SiteDownloaderError
|
||||||
from bdfr.resource import Resource
|
from bdfr.resource import Resource
|
||||||
from bdfr.site_authenticator import SiteAuthenticator
|
from bdfr.site_authenticator import SiteAuthenticator
|
||||||
|
|
||||||
|
@ -27,7 +27,11 @@ class BaseDownloader(ABC):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
|
def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
|
||||||
|
try:
|
||||||
res = requests.get(url, cookies=cookies, headers=headers)
|
res = requests.get(url, cookies=cookies, headers=headers)
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
|
logger.exception(e)
|
||||||
|
raise SiteDownloaderError(f'Failed to get page {url}')
|
||||||
if res.status_code != 200:
|
if res.status_code != 200:
|
||||||
raise ResourceNotFound(f'Server responded with {res.status_code} to {url}')
|
raise ResourceNotFound(f'Server responded with {res.status_code} to {url}')
|
||||||
return res
|
return res
|
||||||
|
|
Loading…
Reference in a new issue