CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-06-21 11:40:21 +12:00
Code Issues Releases Wiki Activity

Catch requests errors in site downloaders

Browse source
This commit is contained in:
Serene-Arc 2021-04-23 20:17:06 +10:00
parent d960bc0b7b
commit 6767777944
1 changed files with 6 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
bdfr/site_downloaders/base_downloader.py
View file

@ -8,7 +8,7 @@ from typing import Optional
import requests
from praw.models import Submission
from bdfr.exceptions import ResourceNotFound
from bdfr.exceptions import ResourceNotFound, SiteDownloaderError
from bdfr.resource import Resource
from bdfr.site_authenticator import SiteAuthenticator
@ -27,7 +27,11 @@ class BaseDownloader(ABC):
@staticmethod
def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
res = requests.get(url, cookies=cookies, headers=headers)
try:
res = requests.get(url, cookies=cookies, headers=headers)
except requests.exceptions.RequestException as e:
logger.exception(e)
raise SiteDownloaderError(f'Failed to get page {url}')
if res.status_code != 200:
raise ResourceNotFound(f'Server responded with {res.status_code} to {url}')
return res