Merge pull request #383 from aliparlakci/development

# v2.1.1
## Changelog
  
- OAuth flow modified for Docker environment (#368)
- Fix Youtube downloader crash on links with no downloadable media (#377)
- Fix 404 errors with RedGifs links that were migrated from Gfycat (#374)

.github/workflows/publish.yml

@@ -27,3 +27,9 @@ jobs:
       run: |
         python setup.py sdist bdist_wheel
         twine upload dist/*
+    
+    - name: Upload coverage report
+      uses: actions/upload-artifact@v2
+      with:
+        name: dist
+        path: dist/

bdfr/oauth2.py

@@ -70,8 +70,8 @@ class OAuth2Authenticator:
     def receive_connection() -> socket.socket:
         server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        server.bind(('localhost', 7634))
-        logger.log(9, 'Server listening on localhost:7634')
+        server.bind(('0.0.0.0', 7634))
+        logger.log(9, 'Server listening on 0.0.0.0:7634')
 
         server.listen(1)
         client = server.accept()[0]

bdfr/site_downloaders/gfycat.py

@@ -27,6 +27,7 @@ class Gfycat(Redgifs):
 
         response = Gfycat.retrieve_url(url)
         if re.search(r'(redgifs|gifdeliverynetwork)', response.url):
+            url = url.lower()  # Fixes error with old gfycat/redgifs links
             return Redgifs._get_link(url)
 
         soup = BeautifulSoup(response.text, 'html.parser')

bdfr/site_downloaders/youtube.py

@@ -8,7 +8,7 @@ from typing import Optional
 import youtube_dl
 from praw.models import Submission
 
-from bdfr.exceptions import SiteDownloaderError
+from bdfr.exceptions import (NotADownloadableLinkError, SiteDownloaderError)
 from bdfr.resource import Resource
 from bdfr.site_authenticator import SiteAuthenticator
 from bdfr.site_downloaders.base_downloader import BaseDownloader
@@ -43,7 +43,12 @@ class Youtube(BaseDownloader):
             except youtube_dl.DownloadError as e:
                 raise SiteDownloaderError(f'Youtube download failed: {e}')
 
-            downloaded_file = list(download_path.iterdir())[0]
+            downloaded_file = None
+            downloaded_files = list(download_path.iterdir())
+            if len(downloaded_files) > 0:
+                downloaded_file = downloaded_files[0]
+            else:
+                raise NotADownloadableLinkError(f"No media exists in the URL {self.post.url}")
             extension = downloaded_file.suffix
             with open(downloaded_file, 'rb') as file:
                 content = file.read()

devscripts/configure.ps1

@@ -1,2 +1,5 @@
-copy .\\bdfr\\default_config.cfg .\\test_config.cfg
-echo "`nuser_token = $env:REDDIT_TOKEN" >> ./test_config.cfg
+if (-not ([string]::IsNullOrEmpty($env:REDDIT_TOKEN)))
+{
+    copy .\\bdfr\\default_config.cfg .\\test_config.cfg
+    echo "`nuser_token = $env:REDDIT_TOKEN" >> ./test_config.cfg
+}

devscripts/configure.sh

@@ -1,2 +1,5 @@
-cp ./bdfr/default_config.cfg ./test_config.cfg
-echo -e "\nuser_token = $REDDIT_TOKEN" >> ./test_config.cfg
+if [ ! -z "$REDDIT_TOKEN" ]
+then
+    cp ./bdfr/default_config.cfg ./test_config.cfg
+    echo -e "\nuser_token = $REDDIT_TOKEN" >> ./test_config.cfg
+fi

setup.cfg

@@ -4,7 +4,7 @@ description_file = README.md
 description_content_type = text/markdown
 home_page = https://github.com/aliparlakci/bulk-downloader-for-reddit
 keywords = reddit, download, archive
-version = 2.1.0
+version = 2.1.1
 author = Ali Parlakci
 author_email = parlakciali@gmail.com
 maintainer = Serene Arc

tests/site_downloaders/test_gfycat.py

@@ -14,6 +14,7 @@ from bdfr.site_downloaders.gfycat import Gfycat
     ('https://gfycat.com/definitivecaninecrayfish', 'https://giant.gfycat.com/DefinitiveCanineCrayfish.mp4'),
     ('https://gfycat.com/dazzlingsilkyiguana', 'https://giant.gfycat.com/DazzlingSilkyIguana.mp4'),
     ('https://gfycat.com/webbedimpurebutterfly', 'https://thumbs2.redgifs.com/WebbedImpureButterfly.mp4'),
+    ('https://gfycat.com/CornyLoathsomeHarrierhawk', 'https://thumbs2.redgifs.com/CornyLoathsomeHarrierhawk.mp4')
 ))
 def test_get_link(test_url: str, expected_url: str):
     result = Gfycat._get_link(test_url)

tests/site_downloaders/test_youtube.py

@@ -5,6 +5,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
+from bdfr.exceptions import NotADownloadableLinkError
 from bdfr.resource import Resource
 from bdfr.site_downloaders.youtube import Youtube
 
@@ -15,7 +16,7 @@ from bdfr.site_downloaders.youtube import Youtube
     ('https://www.youtube.com/watch?v=uSm2VDgRIUs', 'f70b704b4b78b9bb5cd032bfc26e4971'),
     ('https://www.youtube.com/watch?v=m-tKnjFwleU', '30314930d853afff8ebc7d8c36a5b833'),
 ))
-def test_find_resources(test_url: str, expected_hash: str):
+def test_find_resources_good(test_url: str, expected_hash: str):
     test_submission = MagicMock()
     test_submission.url = test_url
     downloader = Youtube(test_submission)
@@ -24,3 +25,15 @@ def test_find_resources(test_url: str, expected_hash: str):
     assert isinstance(resources[0], Resource)
     resources[0].download(120)
     assert resources[0].hash.hexdigest() == expected_hash
+
+
+@pytest.mark.online
+@pytest.mark.parametrize(('test_url'), (
+    ('https://www.polygon.com/disney-plus/2020/5/14/21249881/gargoyles-animated-series-disney-plus-greg-weisman-interview-oj-simpson-goliath-chronicles'),
+))
+def test_find_resources_bad(test_url: str):
+    test_submission = MagicMock()
+    test_submission.url = test_url
+    downloader = Youtube(test_submission)
+    with pytest.raises(NotADownloadableLinkError):
+        downloader.find_resources()