Merge pull request #383 from aliparlakci/development
# v2.1.1 ## Changelog - OAuth flow modified for Docker environment (#368) - Fix Youtube downloader crash on links with no downloadable media (#377) - Fix 404 errors with RedGifs links that were migrated from Gfycat (#374)
This commit is contained in:
commit
aa750f9ab4
6
.github/workflows/publish.yml
vendored
6
.github/workflows/publish.yml
vendored
|
@ -27,3 +27,9 @@ jobs:
|
||||||
run: |
|
run: |
|
||||||
python setup.py sdist bdist_wheel
|
python setup.py sdist bdist_wheel
|
||||||
twine upload dist/*
|
twine upload dist/*
|
||||||
|
|
||||||
|
- name: Upload coverage report
|
||||||
|
uses: actions/upload-artifact@v2
|
||||||
|
with:
|
||||||
|
name: dist
|
||||||
|
path: dist/
|
||||||
|
|
|
@ -70,8 +70,8 @@ class OAuth2Authenticator:
|
||||||
def receive_connection() -> socket.socket:
|
def receive_connection() -> socket.socket:
|
||||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
server.bind(('localhost', 7634))
|
server.bind(('0.0.0.0', 7634))
|
||||||
logger.log(9, 'Server listening on localhost:7634')
|
logger.log(9, 'Server listening on 0.0.0.0:7634')
|
||||||
|
|
||||||
server.listen(1)
|
server.listen(1)
|
||||||
client = server.accept()[0]
|
client = server.accept()[0]
|
||||||
|
|
|
@ -27,6 +27,7 @@ class Gfycat(Redgifs):
|
||||||
|
|
||||||
response = Gfycat.retrieve_url(url)
|
response = Gfycat.retrieve_url(url)
|
||||||
if re.search(r'(redgifs|gifdeliverynetwork)', response.url):
|
if re.search(r'(redgifs|gifdeliverynetwork)', response.url):
|
||||||
|
url = url.lower() # Fixes error with old gfycat/redgifs links
|
||||||
return Redgifs._get_link(url)
|
return Redgifs._get_link(url)
|
||||||
|
|
||||||
soup = BeautifulSoup(response.text, 'html.parser')
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||||||
|
|
|
@ -8,7 +8,7 @@ from typing import Optional
|
||||||
import youtube_dl
|
import youtube_dl
|
||||||
from praw.models import Submission
|
from praw.models import Submission
|
||||||
|
|
||||||
from bdfr.exceptions import SiteDownloaderError
|
from bdfr.exceptions import (NotADownloadableLinkError, SiteDownloaderError)
|
||||||
from bdfr.resource import Resource
|
from bdfr.resource import Resource
|
||||||
from bdfr.site_authenticator import SiteAuthenticator
|
from bdfr.site_authenticator import SiteAuthenticator
|
||||||
from bdfr.site_downloaders.base_downloader import BaseDownloader
|
from bdfr.site_downloaders.base_downloader import BaseDownloader
|
||||||
|
@ -43,7 +43,12 @@ class Youtube(BaseDownloader):
|
||||||
except youtube_dl.DownloadError as e:
|
except youtube_dl.DownloadError as e:
|
||||||
raise SiteDownloaderError(f'Youtube download failed: {e}')
|
raise SiteDownloaderError(f'Youtube download failed: {e}')
|
||||||
|
|
||||||
downloaded_file = list(download_path.iterdir())[0]
|
downloaded_file = None
|
||||||
|
downloaded_files = list(download_path.iterdir())
|
||||||
|
if len(downloaded_files) > 0:
|
||||||
|
downloaded_file = downloaded_files[0]
|
||||||
|
else:
|
||||||
|
raise NotADownloadableLinkError(f"No media exists in the URL {self.post.url}")
|
||||||
extension = downloaded_file.suffix
|
extension = downloaded_file.suffix
|
||||||
with open(downloaded_file, 'rb') as file:
|
with open(downloaded_file, 'rb') as file:
|
||||||
content = file.read()
|
content = file.read()
|
||||||
|
|
|
@ -1,2 +1,5 @@
|
||||||
copy .\\bdfr\\default_config.cfg .\\test_config.cfg
|
if (-not ([string]::IsNullOrEmpty($env:REDDIT_TOKEN)))
|
||||||
echo "`nuser_token = $env:REDDIT_TOKEN" >> ./test_config.cfg
|
{
|
||||||
|
copy .\\bdfr\\default_config.cfg .\\test_config.cfg
|
||||||
|
echo "`nuser_token = $env:REDDIT_TOKEN" >> ./test_config.cfg
|
||||||
|
}
|
|
@ -1,2 +1,5 @@
|
||||||
cp ./bdfr/default_config.cfg ./test_config.cfg
|
if [ ! -z "$REDDIT_TOKEN" ]
|
||||||
echo -e "\nuser_token = $REDDIT_TOKEN" >> ./test_config.cfg
|
then
|
||||||
|
cp ./bdfr/default_config.cfg ./test_config.cfg
|
||||||
|
echo -e "\nuser_token = $REDDIT_TOKEN" >> ./test_config.cfg
|
||||||
|
fi
|
|
@ -4,7 +4,7 @@ description_file = README.md
|
||||||
description_content_type = text/markdown
|
description_content_type = text/markdown
|
||||||
home_page = https://github.com/aliparlakci/bulk-downloader-for-reddit
|
home_page = https://github.com/aliparlakci/bulk-downloader-for-reddit
|
||||||
keywords = reddit, download, archive
|
keywords = reddit, download, archive
|
||||||
version = 2.1.0
|
version = 2.1.1
|
||||||
author = Ali Parlakci
|
author = Ali Parlakci
|
||||||
author_email = parlakciali@gmail.com
|
author_email = parlakciali@gmail.com
|
||||||
maintainer = Serene Arc
|
maintainer = Serene Arc
|
||||||
|
|
|
@ -14,6 +14,7 @@ from bdfr.site_downloaders.gfycat import Gfycat
|
||||||
('https://gfycat.com/definitivecaninecrayfish', 'https://giant.gfycat.com/DefinitiveCanineCrayfish.mp4'),
|
('https://gfycat.com/definitivecaninecrayfish', 'https://giant.gfycat.com/DefinitiveCanineCrayfish.mp4'),
|
||||||
('https://gfycat.com/dazzlingsilkyiguana', 'https://giant.gfycat.com/DazzlingSilkyIguana.mp4'),
|
('https://gfycat.com/dazzlingsilkyiguana', 'https://giant.gfycat.com/DazzlingSilkyIguana.mp4'),
|
||||||
('https://gfycat.com/webbedimpurebutterfly', 'https://thumbs2.redgifs.com/WebbedImpureButterfly.mp4'),
|
('https://gfycat.com/webbedimpurebutterfly', 'https://thumbs2.redgifs.com/WebbedImpureButterfly.mp4'),
|
||||||
|
('https://gfycat.com/CornyLoathsomeHarrierhawk', 'https://thumbs2.redgifs.com/CornyLoathsomeHarrierhawk.mp4')
|
||||||
))
|
))
|
||||||
def test_get_link(test_url: str, expected_url: str):
|
def test_get_link(test_url: str, expected_url: str):
|
||||||
result = Gfycat._get_link(test_url)
|
result = Gfycat._get_link(test_url)
|
||||||
|
|
|
@ -5,6 +5,7 @@ from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from bdfr.exceptions import NotADownloadableLinkError
|
||||||
from bdfr.resource import Resource
|
from bdfr.resource import Resource
|
||||||
from bdfr.site_downloaders.youtube import Youtube
|
from bdfr.site_downloaders.youtube import Youtube
|
||||||
|
|
||||||
|
@ -15,7 +16,7 @@ from bdfr.site_downloaders.youtube import Youtube
|
||||||
('https://www.youtube.com/watch?v=uSm2VDgRIUs', 'f70b704b4b78b9bb5cd032bfc26e4971'),
|
('https://www.youtube.com/watch?v=uSm2VDgRIUs', 'f70b704b4b78b9bb5cd032bfc26e4971'),
|
||||||
('https://www.youtube.com/watch?v=m-tKnjFwleU', '30314930d853afff8ebc7d8c36a5b833'),
|
('https://www.youtube.com/watch?v=m-tKnjFwleU', '30314930d853afff8ebc7d8c36a5b833'),
|
||||||
))
|
))
|
||||||
def test_find_resources(test_url: str, expected_hash: str):
|
def test_find_resources_good(test_url: str, expected_hash: str):
|
||||||
test_submission = MagicMock()
|
test_submission = MagicMock()
|
||||||
test_submission.url = test_url
|
test_submission.url = test_url
|
||||||
downloader = Youtube(test_submission)
|
downloader = Youtube(test_submission)
|
||||||
|
@ -24,3 +25,15 @@ def test_find_resources(test_url: str, expected_hash: str):
|
||||||
assert isinstance(resources[0], Resource)
|
assert isinstance(resources[0], Resource)
|
||||||
resources[0].download(120)
|
resources[0].download(120)
|
||||||
assert resources[0].hash.hexdigest() == expected_hash
|
assert resources[0].hash.hexdigest() == expected_hash
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.online
|
||||||
|
@pytest.mark.parametrize(('test_url'), (
|
||||||
|
('https://www.polygon.com/disney-plus/2020/5/14/21249881/gargoyles-animated-series-disney-plus-greg-weisman-interview-oj-simpson-goliath-chronicles'),
|
||||||
|
))
|
||||||
|
def test_find_resources_bad(test_url: str):
|
||||||
|
test_submission = MagicMock()
|
||||||
|
test_submission.url = test_url
|
||||||
|
downloader = Youtube(test_submission)
|
||||||
|
with pytest.raises(NotADownloadableLinkError):
|
||||||
|
downloader.find_resources()
|
||||||
|
|
Loading…
Reference in a new issue