CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-06-22 04:00:20 +12:00
Code Issues Releases Wiki Activity

Merge pull request #427 from Serene-Arc/bug_fix_386

Browse source 
Add a file blacklist to the direct downloader
This commit is contained in:
Ali Parlakçı 2021-05-25 13:54:41 +03:00 committed by GitHub
parent 0b5f18430f fef2fc864b
commit 346df4726d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 39 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
bdfr/site_downloaders/download_factory.py
View file

@ -21,10 +21,11 @@ from bdfr.site_downloaders.youtube import Youtube
class DownloadFactory:
@staticmethod
def pull_lever(url: str) -> Type[BaseDownloader]:
sanitised_url = DownloadFactory._sanitise_url(url)
sanitised_url = DownloadFactory.sanitise_url(url)
if re.match(r'(i\.)?imgur.*\.gifv$', sanitised_url):
return Imgur
elif re.match(r'.*/.*\.\w{3,4}(\?[\w;&=]*)?$', sanitised_url):
elif re.match(r'.*/.*\.\w{3,4}(\?[\w;&=]*)?$', sanitised_url) and \
not DownloadFactory.is_web_resource(sanitised_url):
return Direct
elif re.match(r'erome\.com.*', sanitised_url):
return Erome
@ -49,9 +50,29 @@ class DownloadFactory:
f'No downloader module exists for url {url}')
@staticmethod
def _sanitise_url(url: str) -> str:
def sanitise_url(url: str) -> str:
beginning_regex = re.compile(r'\s*(www\.?)?')
split_url = urllib.parse.urlsplit(url)
split_url = split_url.netloc + split_url.path
split_url = re.sub(beginning_regex, '', split_url)
return split_url
@staticmethod
def is_web_resource(url: str) -> bool:
web_extensions = (
'asp',
'aspx',
'cfm',
'cfml',
'css',
'htm',
'html',
'js',
'php',
'php3',
'xhtml',
)
if re.match(rf'(?i).*/.*\.({"|".join(web_extensions)})$', url):
return True
else:
return False

17
tests/site_downloaders/test_download_factory.py
View file

@ -69,6 +69,19 @@ def test_factory_lever_bad(test_url: str):
('https://youtube.com/watch?v=Gv8Wz74FjVA', 'youtube.com/watch'),
('https://i.imgur.com/BuzvZwb.gifv', 'i.imgur.com/BuzvZwb.gifv'),
))
def test_sanitise_urll(test_url: str, expected: str):
result = DownloadFactory._sanitise_url(test_url)
def test_sanitise_url(test_url: str, expected: str):
result = DownloadFactory.sanitise_url(test_url)
assert result == expected
@pytest.mark.parametrize(('test_url', 'expected'), (
('www.example.com/test.asp', True),
('www.example.com/test.html', True),
('www.example.com/test.js', True),
('www.example.com/test.xhtml', True),
('www.example.com/test.mp4', False),
('www.example.com/test.png', False),
))
def test_is_web_resource(test_url: str, expected: bool):
result = DownloadFactory.is_web_resource(test_url)
assert result == expected