From 5288b79d1bbd70b0dd29468ec8d5a63afeddf46b Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Thu, 9 Dec 2021 13:04:11 +1000 Subject: [PATCH 1/4] Add test for time checking --- tests/test_connector.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/test_connector.py b/tests/test_connector.py index 35d0b2e..9fe58f2 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # coding=utf-8 - +from datetime import datetime, timedelta from pathlib import Path from typing import Iterator from unittest.mock import MagicMock @@ -195,6 +195,39 @@ def test_get_subreddit_normal( assert not any([isinstance(m, MagicMock) for m in results]) +@pytest.mark.online +@pytest.mark.reddit +@pytest.mark.parametrize(('test_time', 'test_delta'), ( + ('hour', timedelta(hours=1)), + ('day', timedelta(days=1)), + ('week', timedelta(days=7)), + ('month', timedelta(days=31)), + ('year', timedelta(days=365)), +)) +def test_get_subreddit_time_verification( + test_time: str, + test_delta: timedelta, + downloader_mock: MagicMock, + reddit_instance: praw.Reddit, +): + downloader_mock.args.limit = 10 + downloader_mock.args.sort = 'top' + downloader_mock.args.time = test_time + downloader_mock.time_filter = RedditConnector.create_time_filter(downloader_mock) + downloader_mock.sort_filter = RedditConnector.create_sort_filter(downloader_mock) + downloader_mock.determine_sort_function.return_value = RedditConnector.determine_sort_function(downloader_mock) + downloader_mock.args.subreddit = ['all'] + downloader_mock.reddit_instance = reddit_instance + results = RedditConnector.get_subreddits(downloader_mock) + results = [sub for res1 in results for sub in res1] + assert all([isinstance(res1, praw.models.Submission) for res1 in results]) + nowtime = datetime.now() + for r in results: + result_time = datetime.fromtimestamp(r.created_utc) + time_diff = nowtime - result_time + assert time_diff < test_delta + + @pytest.mark.online @pytest.mark.reddit @pytest.mark.parametrize(('test_subreddits', 'search_term', 'limit', 'time_filter', 'max_expected_len'), ( From 36ff95de6b8eee13c901ce37b545e826fe36a12e Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Sun, 19 Dec 2021 13:44:24 +1000 Subject: [PATCH 2/4] Add Patreon image support --- bdfr/site_downloaders/download_factory.py | 2 ++ tests/site_downloaders/test_download_factory.py | 1 + tests/site_downloaders/test_gallery.py | 8 ++++++++ 3 files changed, 11 insertions(+) diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py index 91489a0..49dba5f 100644 --- a/bdfr/site_downloaders/download_factory.py +++ b/bdfr/site_downloaders/download_factory.py @@ -33,6 +33,8 @@ class DownloadFactory: return Erome elif re.match(r'reddit\.com/gallery/.*', sanitised_url): return Gallery + elif re.match(r'patreon\.com.*', sanitised_url): + return Gallery elif re.match(r'gfycat\.', sanitised_url): return Gfycat elif re.match(r'(m\.)?imgur.*', sanitised_url): diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py index 441b554..134396c 100644 --- a/tests/site_downloaders/test_download_factory.py +++ b/tests/site_downloaders/test_download_factory.py @@ -47,6 +47,7 @@ from bdfr.site_downloaders.youtube import Youtube ('https://vimeo.com/channels/31259/53576664', YtdlpFallback), ('http://video.pbs.org/viralplayer/2365173446/', YtdlpFallback), ('https://www.pornhub.com/view_video.php?viewkey=ph5a2ee0461a8d0', PornHub), + ('https://www.patreon.com/posts/minecart-track-59346560', Gallery), )) def test_factory_lever_good(test_submission_url: str, expected_class: BaseDownloader, reddit_instance: praw.Reddit): result = DownloadFactory.pull_lever(test_submission_url) diff --git a/tests/site_downloaders/test_gallery.py b/tests/site_downloaders/test_gallery.py index 08eea91..e9c401f 100644 --- a/tests/site_downloaders/test_gallery.py +++ b/tests/site_downloaders/test_gallery.py @@ -57,6 +57,14 @@ def test_gallery_get_links(test_ids: list[dict], expected: set[str]): '65163f685fb28c5b776e0e77122718be', '2a337eb5b13c34d3ca3f51b5db7c13e9', }), + ('rb3ub6', { # patreon post + '748a976c6cedf7ea85b6f90e7cb685c7', + '839796d7745e88ced6355504e1f74508', + 'bcdb740367d0f19f97a77e614b48a42d', + '0f230b8c4e5d103d35a773fab9814ec3', + 'e5192d6cb4f84c4f4a658355310bf0f9', + '91cbe172cd8ccbcf049fcea4204eb979', + }) )) def test_gallery_download(test_submission_id: str, expected_hashes: set[str], reddit_instance: praw.Reddit): test_submission = reddit_instance.submission(id=test_submission_id) From af0a545c163c93090f56a3f1ebafdaef7b1f4c45 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Mon, 20 Dec 2021 20:43:09 +1000 Subject: [PATCH 3/4] Catch additional error in Gallery --- bdfr/site_downloaders/gallery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bdfr/site_downloaders/gallery.py b/bdfr/site_downloaders/gallery.py index 158e338..eeb9e0f 100644 --- a/bdfr/site_downloaders/gallery.py +++ b/bdfr/site_downloaders/gallery.py @@ -24,7 +24,7 @@ class Gallery(BaseDownloader): except (AttributeError, TypeError): try: image_urls = self._get_links(self.post.crosspost_parent_list[0]['gallery_data']['items']) - except (AttributeError, IndexError, TypeError): + except (AttributeError, IndexError, TypeError, KeyError): logger.error(f'Could not find gallery data in submission {self.post.id}') logger.exception('Gallery image find failure') raise SiteDownloaderError('No images found in Reddit gallery') From e564870cd670b2b9de85542c5a0fde2ccd48f5d1 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Mon, 20 Dec 2021 20:50:47 +1000 Subject: [PATCH 4/4] Increase version number --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index c476c0e..198ebe7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ description_file = README.md description_content_type = text/markdown home_page = https://github.com/aliparlakci/bulk-downloader-for-reddit keywords = reddit, download, archive -version = 2.5.1 +version = 2.5.2 author = Ali Parlakci author_email = parlakciali@gmail.com maintainer = Serene Arc