RUF013
This commit is contained in:
parent
fb6ae392b1
commit
ca2408b6e1
|
@ -2,6 +2,7 @@
|
|||
|
||||
import logging
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from bdfr.resource import Resource
|
||||
|
||||
|
@ -9,7 +10,11 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
|
||||
class DownloadFilter:
|
||||
def __init__(self, excluded_extensions: list[str] = None, excluded_domains: list[str] = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
excluded_extensions: Optional[list[str]] = None,
|
||||
excluded_domains: Optional[list[str]] = None,
|
||||
) -> None:
|
||||
self.excluded_extensions = excluded_extensions
|
||||
self.excluded_domains = excluded_domains
|
||||
|
||||
|
|
|
@ -19,7 +19,11 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class Resource:
|
||||
def __init__(
|
||||
self, source_submission: Submission, url: str, download_function: Callable, extension: str = None
|
||||
self,
|
||||
source_submission: Submission,
|
||||
url: str,
|
||||
download_function: Callable,
|
||||
extension: Optional[str] = None,
|
||||
) -> None:
|
||||
self.source_submission = source_submission
|
||||
self.content: Optional[bytes] = None
|
||||
|
|
|
@ -25,7 +25,11 @@ class BaseDownloader(ABC):
|
|||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
|
||||
def retrieve_url(
|
||||
url: str,
|
||||
cookies: Optional[dict] = None,
|
||||
headers: Optional[dict] = None,
|
||||
) -> requests.Response:
|
||||
try:
|
||||
res = requests.get(url, cookies=cookies, headers=headers, timeout=10)
|
||||
except requests.exceptions.RequestException as e:
|
||||
|
@ -39,7 +43,12 @@ class BaseDownloader(ABC):
|
|||
return res
|
||||
|
||||
@staticmethod
|
||||
def post_url(url: str, cookies: dict = None, headers: dict = None, payload: dict = None) -> requests.Response:
|
||||
def post_url(
|
||||
url: str,
|
||||
cookies: Optional[dict] = None,
|
||||
headers: Optional[dict] = None,
|
||||
payload: Optional[dict] = None,
|
||||
) -> requests.Response:
|
||||
try:
|
||||
res = requests.post(url, cookies=cookies, headers=headers, json=payload, timeout=10)
|
||||
except requests.exceptions.RequestException as e:
|
||||
|
@ -53,7 +62,7 @@ class BaseDownloader(ABC):
|
|||
return res
|
||||
|
||||
@staticmethod
|
||||
def head_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
|
||||
def head_url(url: str, cookies: Optional[dict] = None, headers: Optional[dict] = None) -> requests.Response:
|
||||
try:
|
||||
res = requests.head(url, cookies=cookies, headers=headers, timeout=10)
|
||||
except requests.exceptions.RequestException as e:
|
||||
|
|
Loading…
Reference in a new issue