CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-06-28 02:50:27 +12:00
Code Issues Releases Wiki Activity

Catch requests errors in site downloaders

Browse source
This commit is contained in:
Serene-Arc 2021-04-23 20:17:06 +10:00
parent d960bc0b7b
commit 6767777944
1 changed files with 6 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
bdfr/site_downloaders/base_downloader.py
View file

@ -8,7 +8,7 @@ from typing import Optional
import requests import requests
from praw.models import Submission from praw.models import Submission
from bdfr.exceptions import ResourceNotFound from bdfr.exceptions import ResourceNotFound, SiteDownloaderError
from bdfr.resource import Resource from bdfr.resource import Resource
from bdfr.site_authenticator import SiteAuthenticator from bdfr.site_authenticator import SiteAuthenticator
@ -27,7 +27,11 @@ class BaseDownloader(ABC):
@staticmethod @staticmethod
def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response: def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
res = requests.get(url, cookies=cookies, headers=headers) try:
res = requests.get(url, cookies=cookies, headers=headers)
except requests.exceptions.RequestException as e:
logger.exception(e)
raise SiteDownloaderError(f'Failed to get page {url}')
if res.status_code != 200: if res.status_code != 200:
raise ResourceNotFound(f'Server responded with {res.status_code} to {url}') raise ResourceNotFound(f'Server responded with {res.status_code} to {url}')
return res return res