1
0
Fork 0
mirror of synced 2024-06-02 18:34:37 +12:00
This commit is contained in:
OMEGARAZER 2023-06-20 17:48:28 -04:00
parent fb6ae392b1
commit ca2408b6e1
No known key found for this signature in database
GPG key ID: D89925310D306E35
3 changed files with 23 additions and 5 deletions

View file

@ -2,6 +2,7 @@
import logging
import re
from typing import Optional
from bdfr.resource import Resource
@ -9,7 +10,11 @@ logger = logging.getLogger(__name__)
class DownloadFilter:
def __init__(self, excluded_extensions: list[str] = None, excluded_domains: list[str] = None) -> None:
def __init__(
self,
excluded_extensions: Optional[list[str]] = None,
excluded_domains: Optional[list[str]] = None,
) -> None:
self.excluded_extensions = excluded_extensions
self.excluded_domains = excluded_domains

View file

@ -19,7 +19,11 @@ logger = logging.getLogger(__name__)
class Resource:
def __init__(
self, source_submission: Submission, url: str, download_function: Callable, extension: str = None
self,
source_submission: Submission,
url: str,
download_function: Callable,
extension: Optional[str] = None,
) -> None:
self.source_submission = source_submission
self.content: Optional[bytes] = None

View file

@ -25,7 +25,11 @@ class BaseDownloader(ABC):
raise NotImplementedError
@staticmethod
def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
def retrieve_url(
url: str,
cookies: Optional[dict] = None,
headers: Optional[dict] = None,
) -> requests.Response:
try:
res = requests.get(url, cookies=cookies, headers=headers, timeout=10)
except requests.exceptions.RequestException as e:
@ -39,7 +43,12 @@ class BaseDownloader(ABC):
return res
@staticmethod
def post_url(url: str, cookies: dict = None, headers: dict = None, payload: dict = None) -> requests.Response:
def post_url(
url: str,
cookies: Optional[dict] = None,
headers: Optional[dict] = None,
payload: Optional[dict] = None,
) -> requests.Response:
try:
res = requests.post(url, cookies=cookies, headers=headers, json=payload, timeout=10)
except requests.exceptions.RequestException as e:
@ -53,7 +62,7 @@ class BaseDownloader(ABC):
return res
@staticmethod
def head_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
def head_url(url: str, cookies: Optional[dict] = None, headers: Optional[dict] = None) -> requests.Response:
try:
res = requests.head(url, cookies=cookies, headers=headers, timeout=10)
except requests.exceptions.RequestException as e: