From 99fe3312a4f343fb45b6d55cf21e54bb10f9dd33 Mon Sep 17 00:00:00 2001 From: BlipRanger <1860540+BlipRanger@users.noreply.github.com> Date: Wed, 12 May 2021 10:18:02 -0400 Subject: [PATCH 01/30] Bind socket to '0.0.0.0' rather than 'localhost' to allow for more flexible OAuth connection. --- bdfr/oauth2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bdfr/oauth2.py b/bdfr/oauth2.py index 6b27599..bd60c9b 100644 --- a/bdfr/oauth2.py +++ b/bdfr/oauth2.py @@ -70,8 +70,8 @@ class OAuth2Authenticator: def receive_connection() -> socket.socket: server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server.bind(('localhost', 7634)) - logger.log(9, 'Server listening on localhost:7634') + server.bind(('0.0.0.0', 7634)) + logger.log(9, 'Server listening on 0.0.0.0:7634') server.listen(1) client = server.accept()[0] From dbd0c6cd424650d76973c283a82793aa4a72d7f6 Mon Sep 17 00:00:00 2001 From: BlipRanger <1860540+BlipRanger@users.noreply.github.com> Date: Mon, 25 Apr 2022 12:09:09 -0400 Subject: [PATCH 02/30] Add support for v.reddit links. --- bdfr/site_downloaders/download_factory.py | 3 + bdfr/site_downloaders/vreddit.py | 77 +++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 bdfr/site_downloaders/vreddit.py diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py index 49dba5f..2c7ef21 100644 --- a/bdfr/site_downloaders/download_factory.py +++ b/bdfr/site_downloaders/download_factory.py @@ -18,6 +18,7 @@ from bdfr.site_downloaders.redgifs import Redgifs from bdfr.site_downloaders.self_post import SelfPost from bdfr.site_downloaders.vidble import Vidble from bdfr.site_downloaders.youtube import Youtube +from bdfr.site_downloaders.vreddit import vreddit class DownloadFactory: @@ -47,6 +48,8 @@ class DownloadFactory: return Youtube elif re.match(r'i\.redd\.it.*', sanitised_url): return Direct + elif re.match(r'v\.redd\.it.*', sanitised_url): + return vreddit elif re.match(r'pornhub\.com.*', sanitised_url): return PornHub elif re.match(r'vidble\.com', sanitised_url): diff --git a/bdfr/site_downloaders/vreddit.py b/bdfr/site_downloaders/vreddit.py new file mode 100644 index 0000000..945ee93 --- /dev/null +++ b/bdfr/site_downloaders/vreddit.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +import logging +import tempfile +from pathlib import Path +from typing import Callable, Optional + +import yt_dlp +from praw.models import Submission + +from bdfr.exceptions import NotADownloadableLinkError, SiteDownloaderError +from bdfr.resource import Resource +from bdfr.site_authenticator import SiteAuthenticator +from bdfr.site_downloaders.base_downloader import BaseDownloader + +logger = logging.getLogger(__name__) + + +class vreddit(BaseDownloader): + def __init__(self, post: Submission): + super().__init__(post) + + def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]: + ytdl_options = { + 'playlistend': 1, + 'nooverwrites': True, + } + download_function = self._download_video(ytdl_options) + extension = self.get_video_attributes(self.post.url)['ext'] + res = Resource(self.post, self.post.url, download_function, extension) + return [res] + + def _download_video(self, ytdl_options: dict) -> Callable: + yt_logger = logging.getLogger('youtube-dl') + yt_logger.setLevel(logging.CRITICAL) + ytdl_options['quiet'] = True + ytdl_options['logger'] = yt_logger + + def download(_: dict) -> bytes: + with tempfile.TemporaryDirectory() as temp_dir: + download_path = Path(temp_dir).resolve() + ytdl_options['outtmpl'] = str(download_path) + '/' + 'test.%(ext)s' + try: + with yt_dlp.YoutubeDL(ytdl_options) as ydl: + ydl.download([self.post.url]) + except yt_dlp.DownloadError as e: + raise SiteDownloaderError(f'Vreddit download failed: {e}') + + downloaded_files = list(download_path.iterdir()) + if len(downloaded_files) > 0: + downloaded_file = downloaded_files[0] + else: + raise NotADownloadableLinkError(f"No media exists in the URL {self.post.url}") + with open(downloaded_file, 'rb') as file: + content = file.read() + return content + return download + + @staticmethod + def get_video_attributes(url: str) -> dict: + yt_logger = logging.getLogger('youtube-dl') + yt_logger.setLevel(logging.CRITICAL) + with yt_dlp.YoutubeDL({'logger': yt_logger, }) as ydl: + try: + result = ydl.extract_info(url, download=False) + except Exception as e: + logger.exception(e) + raise NotADownloadableLinkError(f'Video info extraction failed for {url}') + if 'ext' in result: + return result + else: + try: + result = (result["entries"][0]) + return result + except Exception as e: + logger.exception(e) + raise NotADownloadableLinkError(f'Video info extraction failed for {url}') From d64acc25f52b85eec74a35fbc229fa080f869d0c Mon Sep 17 00:00:00 2001 From: BlipRanger <1860540+BlipRanger@users.noreply.github.com> Date: Mon, 25 Apr 2022 12:53:59 -0400 Subject: [PATCH 03/30] Add tests, fix style. --- bdfr/site_downloaders/download_factory.py | 4 +- bdfr/site_downloaders/vreddit.py | 2 +- .../site_downloaders/test_download_factory.py | 2 + tests/site_downloaders/test_vreddit.py | 39 +++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/site_downloaders/test_vreddit.py diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py index 2c7ef21..f6f4895 100644 --- a/bdfr/site_downloaders/download_factory.py +++ b/bdfr/site_downloaders/download_factory.py @@ -18,7 +18,7 @@ from bdfr.site_downloaders.redgifs import Redgifs from bdfr.site_downloaders.self_post import SelfPost from bdfr.site_downloaders.vidble import Vidble from bdfr.site_downloaders.youtube import Youtube -from bdfr.site_downloaders.vreddit import vreddit +from bdfr.site_downloaders.vreddit import Vreddit class DownloadFactory: @@ -49,7 +49,7 @@ class DownloadFactory: elif re.match(r'i\.redd\.it.*', sanitised_url): return Direct elif re.match(r'v\.redd\.it.*', sanitised_url): - return vreddit + return Vreddit elif re.match(r'pornhub\.com.*', sanitised_url): return PornHub elif re.match(r'vidble\.com', sanitised_url): diff --git a/bdfr/site_downloaders/vreddit.py b/bdfr/site_downloaders/vreddit.py index 945ee93..b1117a0 100644 --- a/bdfr/site_downloaders/vreddit.py +++ b/bdfr/site_downloaders/vreddit.py @@ -16,7 +16,7 @@ from bdfr.site_downloaders.base_downloader import BaseDownloader logger = logging.getLogger(__name__) -class vreddit(BaseDownloader): +class Vreddit(BaseDownloader): def __init__(self, post: Submission): super().__init__(post) diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py index 134396c..b66d17b 100644 --- a/tests/site_downloaders/test_download_factory.py +++ b/tests/site_downloaders/test_download_factory.py @@ -17,6 +17,7 @@ from bdfr.site_downloaders.pornhub import PornHub from bdfr.site_downloaders.redgifs import Redgifs from bdfr.site_downloaders.self_post import SelfPost from bdfr.site_downloaders.youtube import Youtube +from bdfr.site_downloaders.vreddit import Vreddit @pytest.mark.online @@ -48,6 +49,7 @@ from bdfr.site_downloaders.youtube import Youtube ('http://video.pbs.org/viralplayer/2365173446/', YtdlpFallback), ('https://www.pornhub.com/view_video.php?viewkey=ph5a2ee0461a8d0', PornHub), ('https://www.patreon.com/posts/minecart-track-59346560', Gallery), + ('https://v.redd.it/9z1dnk3xr5k61', Vreddit) )) def test_factory_lever_good(test_submission_url: str, expected_class: BaseDownloader, reddit_instance: praw.Reddit): result = DownloadFactory.pull_lever(test_submission_url) diff --git a/tests/site_downloaders/test_vreddit.py b/tests/site_downloaders/test_vreddit.py new file mode 100644 index 0000000..65428b5 --- /dev/null +++ b/tests/site_downloaders/test_vreddit.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +from unittest.mock import MagicMock + +import pytest + +from bdfr.exceptions import NotADownloadableLinkError +from bdfr.resource import Resource +from bdfr.site_downloaders.vreddit import Vreddit + + +@pytest.mark.online +@pytest.mark.slow +@pytest.mark.parametrize(('test_url', 'expected_hash'), ( + ('https://www.reddit.com/user/Xomb_Forever/comments/u5p2kj/hold_up/', '690cffe27a7884196437926c22897216'), +)) +def test_find_resources_good(test_url: str, expected_hash: str): + test_submission = MagicMock() + test_submission.url = test_url + downloader = Vreddit(test_submission) + resources = downloader.find_resources() + assert len(resources) == 1 + assert isinstance(resources[0], Resource) + resources[0].download() + assert resources[0].hash.hexdigest() == expected_hash + + +@pytest.mark.online +@pytest.mark.parametrize('test_url', ( + 'https://www.polygon.com/disney-plus/2020/5/14/21249881/gargoyles-animated-series-disney-plus-greg-weisman' + '-interview-oj-simpson-goliath-chronicles', +)) +def test_find_resources_bad(test_url: str): + test_submission = MagicMock() + test_submission.url = test_url + downloader = Vreddit(test_submission) + with pytest.raises(NotADownloadableLinkError): + downloader.find_resources() From 274407537ef3b2426262ceb8a14a340eb0cc4a0f Mon Sep 17 00:00:00 2001 From: BlipRanger <1860540+BlipRanger@users.noreply.github.com> Date: Mon, 25 Apr 2022 13:02:42 -0400 Subject: [PATCH 04/30] Fix one test --- tests/site_downloaders/test_download_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py index b66d17b..30e19f1 100644 --- a/tests/site_downloaders/test_download_factory.py +++ b/tests/site_downloaders/test_download_factory.py @@ -43,7 +43,7 @@ from bdfr.site_downloaders.vreddit import Vreddit ('https://i.imgur.com/3SKrQfK.jpg?1', Direct), ('https://dynasty-scans.com/system/images_images/000/017/819/original/80215103_p0.png?1612232781', Direct), ('https://m.imgur.com/a/py3RW0j', Imgur), - ('https://v.redd.it/9z1dnk3xr5k61', YtdlpFallback), + ('https://v.redd.it/9z1dnk3xr5k61', Vreddit), ('https://streamable.com/dt46y', YtdlpFallback), ('https://vimeo.com/channels/31259/53576664', YtdlpFallback), ('http://video.pbs.org/viralplayer/2365173446/', YtdlpFallback), From a620ae91a18013d1e172d628d66b65980e16c0e0 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 18 Feb 2022 10:21:52 +1000 Subject: [PATCH 05/30] Add --subscribed option --- bdfr/__main__.py | 1 + bdfr/configuration.py | 1 + bdfr/connector.py | 20 +++++++---- bdfr/default_config.cfg | 2 +- .../test_download_integration.py | 35 ++++++++++--------- tests/test_connector.py | 18 +++++++++- 6 files changed, 52 insertions(+), 25 deletions(-) diff --git a/bdfr/__main__.py b/bdfr/__main__.py index de658de..56ffb0f 100644 --- a/bdfr/__main__.py +++ b/bdfr/__main__.py @@ -23,6 +23,7 @@ _common_options = [ click.option('--saved', is_flag=True, default=None), click.option('--search', default=None, type=str), click.option('--submitted', is_flag=True, default=None), + click.option('--subscribed', is_flag=True, default=None), click.option('--time-format', type=str, default=None), click.option('--upvoted', is_flag=True, default=None), click.option('-L', '--limit', default=None, type=int), diff --git a/bdfr/configuration.py b/bdfr/configuration.py index 81fa3e4..ef24e36 100644 --- a/bdfr/configuration.py +++ b/bdfr/configuration.py @@ -35,6 +35,7 @@ class Configuration(Namespace): self.skip_subreddit: list[str] = [] self.sort: str = 'hot' self.submitted: bool = False + self.subscribed: bool = True self.subreddit: list[str] = [] self.time: str = 'all' self.time_format = None diff --git a/bdfr/connector.py b/bdfr/connector.py index 506e23f..e04d9ef 100644 --- a/bdfr/connector.py +++ b/bdfr/connector.py @@ -243,9 +243,19 @@ class RedditConnector(metaclass=ABCMeta): return set(all_entries) def get_subreddits(self) -> list[praw.models.ListingGenerator]: - if self.args.subreddit: - out = [] - for reddit in self.split_args_input(self.args.subreddit): + out = [] + subscribed_subreddits = set() + if self.args.subscribed: + if self.args.authenticate: + try: + subscribed_subreddits = list(self.reddit_instance.user.subreddits(limit=None)) + subscribed_subreddits = set([s.display_name for s in subscribed_subreddits]) + except prawcore.InsufficientScope: + logger.error('BDFR has insufficient scope to access subreddit lists') + else: + logger.error('Cannot find subscribed subreddits without an authenticated instance') + if self.args.subreddit or subscribed_subreddits: + for reddit in self.split_args_input(self.args.subreddit) | subscribed_subreddits: if reddit == 'friends' and self.authenticated is False: logger.error('Cannot read friends subreddit without an authenticated instance') continue @@ -270,9 +280,7 @@ class RedditConnector(metaclass=ABCMeta): logger.debug(f'Added submissions from subreddit {reddit}') except (errors.BulkDownloaderException, praw.exceptions.PRAWException) as e: logger.error(f'Failed to get submissions for subreddit {reddit}: {e}') - return out - else: - return [] + return out def resolve_user_name(self, in_name: str) -> str: if in_name == 'me': diff --git a/bdfr/default_config.cfg b/bdfr/default_config.cfg index b8039a9..c601152 100644 --- a/bdfr/default_config.cfg +++ b/bdfr/default_config.cfg @@ -1,7 +1,7 @@ [DEFAULT] client_id = U-6gk4ZCh3IeNQ client_secret = 7CZHY6AmKweZME5s50SfDGylaPg -scopes = identity, history, read, save +scopes = identity, history, read, save, mysubreddits backup_log_count = 3 max_wait_time = 120 time_format = ISO \ No newline at end of file diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py index bd53382..75216dd 100644 --- a/tests/integration_tests/test_download_integration.py +++ b/tests/integration_tests/test_download_integration.py @@ -31,23 +31,23 @@ def create_basic_args_for_download_runner(test_args: list[str], run_path: Path): @pytest.mark.reddit @pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests') @pytest.mark.parametrize('test_args', ( - ['-s', 'Mindustry', '-L', 1], - ['-s', 'r/Mindustry', '-L', 1], - ['-s', 'r/mindustry', '-L', 1], - ['-s', 'mindustry', '-L', 1], - ['-s', 'https://www.reddit.com/r/TrollXChromosomes/', '-L', 1], - ['-s', 'r/TrollXChromosomes/', '-L', 1], - ['-s', 'TrollXChromosomes/', '-L', 1], - ['-s', 'trollxchromosomes', '-L', 1], - ['-s', 'trollxchromosomes,mindustry,python', '-L', 1], - ['-s', 'trollxchromosomes, mindustry, python', '-L', 1], - ['-s', 'trollxchromosomes', '-L', 1, '--time', 'day'], - ['-s', 'trollxchromosomes', '-L', 1, '--sort', 'new'], - ['-s', 'trollxchromosomes', '-L', 1, '--time', 'day', '--sort', 'new'], - ['-s', 'trollxchromosomes', '-L', 1, '--search', 'women'], - ['-s', 'trollxchromosomes', '-L', 1, '--time', 'day', '--search', 'women'], - ['-s', 'trollxchromosomes', '-L', 1, '--sort', 'new', '--search', 'women'], - ['-s', 'trollxchromosomes', '-L', 1, '--time', 'day', '--sort', 'new', '--search', 'women'], + ['-s', 'Mindustry', '-L', 3], + ['-s', 'r/Mindustry', '-L', 3], + ['-s', 'r/mindustry', '-L', 3], + ['-s', 'mindustry', '-L', 3], + ['-s', 'https://www.reddit.com/r/TrollXChromosomes/', '-L', 3], + ['-s', 'r/TrollXChromosomes/', '-L', 3], + ['-s', 'TrollXChromosomes/', '-L', 3], + ['-s', 'trollxchromosomes', '-L', 3], + ['-s', 'trollxchromosomes,mindustry,python', '-L', 3], + ['-s', 'trollxchromosomes, mindustry, python', '-L', 3], + ['-s', 'trollxchromosomes', '-L', 3, '--time', 'day'], + ['-s', 'trollxchromosomes', '-L', 3, '--sort', 'new'], + ['-s', 'trollxchromosomes', '-L', 3, '--time', 'day', '--sort', 'new'], + ['-s', 'trollxchromosomes', '-L', 3, '--search', 'women'], + ['-s', 'trollxchromosomes', '-L', 3, '--time', 'day', '--search', 'women'], + ['-s', 'trollxchromosomes', '-L', 3, '--sort', 'new', '--search', 'women'], + ['-s', 'trollxchromosomes', '-L', 3, '--time', 'day', '--sort', 'new', '--search', 'women'], )) def test_cli_download_subreddits(test_args: list[str], tmp_path: Path): runner = CliRunner() @@ -64,6 +64,7 @@ def test_cli_download_subreddits(test_args: list[str], tmp_path: Path): @pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests') @pytest.mark.parametrize('test_args', ( ['-s', 'hentai', '-L', 10, '--search', 'red', '--authenticate'], + ['--authenticate', '--subscribed', '-L', 10], )) def test_cli_download_search_subreddits_authenticated(test_args: list[str], tmp_path: Path): runner = CliRunner() diff --git a/tests/test_connector.py b/tests/test_connector.py index 9fe58f2..3a10757 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -336,13 +336,29 @@ def test_get_user_authenticated_lists( downloader_mock.args.__dict__[test_flag] = True downloader_mock.reddit_instance = authenticated_reddit_instance downloader_mock.args.limit = 10 - downloader_mock._determine_sort_function.return_value = praw.models.Subreddit.hot + downloader_mock.determine_sort_function.return_value = praw.models.Subreddit.hot downloader_mock.sort_filter = RedditTypes.SortType.HOT downloader_mock.args.user = [RedditConnector.resolve_user_name(downloader_mock, 'me')] results = RedditConnector.get_user_data(downloader_mock) assert_all_results_are_submissions_or_comments(10, results) +@pytest.mark.online +@pytest.mark.reddit +@pytest.mark.authenticated +def test_get_subscribed_subreddits(downloader_mock: MagicMock, authenticated_reddit_instance: praw.Reddit): + downloader_mock.reddit_instance = authenticated_reddit_instance + downloader_mock.args.limit = 10 + downloader_mock.args.authenticate = True + downloader_mock.args.subscribed = True + downloader_mock.determine_sort_function.return_value = praw.models.Subreddit.hot + downloader_mock.determine_sort_function.return_value = praw.models.Subreddit.hot + downloader_mock.sort_filter = RedditTypes.SortType.HOT + results = RedditConnector.get_subreddits(downloader_mock) + assert all([isinstance(s, praw.models.ListingGenerator) for s in results]) + assert len(results) > 0 + + @pytest.mark.parametrize(('test_name', 'expected'), ( ('Mindustry', 'Mindustry'), ('Futurology', 'Futurology'), From 90a2eac90dc35f4067789077cb769f2c653c9233 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 18 Feb 2022 12:04:37 +1000 Subject: [PATCH 06/30] Add support for Redgifs images and galleries --- bdfr/site_downloaders/redgifs.py | 31 +++++++++++++---- tests/site_downloaders/test_redgifs.py | 48 ++++++++++++++++++-------- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/bdfr/site_downloaders/redgifs.py b/bdfr/site_downloaders/redgifs.py index a62fedb..12fb24d 100644 --- a/bdfr/site_downloaders/redgifs.py +++ b/bdfr/site_downloaders/redgifs.py @@ -17,11 +17,11 @@ class Redgifs(BaseDownloader): super().__init__(post) def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]: - media_url = self._get_link(self.post.url) - return [Resource(self.post, media_url, Resource.retry_download(media_url), '.mp4')] + media_urls = self._get_link(self.post.url) + return [Resource(self.post, m, Resource.retry_download(m), '.mp4') for m in media_urls] @staticmethod - def _get_link(url: str) -> str: + def _get_link(url: str) -> set[str]: try: redgif_id = re.match(r'.*/(.*?)/?$', url).group(1) except AttributeError: @@ -32,16 +32,33 @@ class Redgifs(BaseDownloader): 'Chrome/90.0.4430.93 Safari/537.36', } - content = Redgifs.retrieve_url(f'https://api.redgifs.com/v1/gfycats/{redgif_id}', headers=headers) + content = Redgifs.retrieve_url(f'https://api.redgifs.com/v2/gifs/{redgif_id}', headers=headers) if content is None: raise SiteDownloaderError('Could not read the page source') try: - out = json.loads(content.text)['gfyItem']['mp4Url'] - except (KeyError, AttributeError): - raise SiteDownloaderError('Failed to find JSON data in page') + response_json = json.loads(content.text) except json.JSONDecodeError as e: raise SiteDownloaderError(f'Received data was not valid JSON: {e}') + out = set() + try: + if response_json['gif']['type'] == 1: # type 1 is a video + out.add(response_json['gif']['urls']['hd']) + elif response_json['gif']['type'] == 2: # type 2 is an image + if response_json['gif']['gallery']: + content = Redgifs.retrieve_url( + f'https://api.redgifs.com/v2/gallery/{response_json["gif"]["gallery"]}', + headers=headers, + ) + response_json = json.loads(content.text) + out = {p['urls']['hd'] for p in response_json['gifs']} + else: + out.add(response_json['gif']['urls']['hd']) + else: + raise KeyError + except (KeyError, AttributeError): + raise SiteDownloaderError('Failed to find JSON data in page') + return out diff --git a/tests/site_downloaders/test_redgifs.py b/tests/site_downloaders/test_redgifs.py index 571f044..a1f571e 100644 --- a/tests/site_downloaders/test_redgifs.py +++ b/tests/site_downloaders/test_redgifs.py @@ -12,30 +12,48 @@ from bdfr.site_downloaders.redgifs import Redgifs @pytest.mark.online @pytest.mark.parametrize(('test_url', 'expected'), ( ('https://redgifs.com/watch/frighteningvictorioussalamander', - 'https://thumbs2.redgifs.com/FrighteningVictoriousSalamander.mp4'), + {'https://thumbs2.redgifs.com/FrighteningVictoriousSalamander.mp4'}), ('https://redgifs.com/watch/springgreendecisivetaruca', - 'https://thumbs2.redgifs.com/SpringgreenDecisiveTaruca.mp4'), + {'https://thumbs2.redgifs.com/SpringgreenDecisiveTaruca.mp4'}), ('https://www.redgifs.com/watch/palegoldenrodrawhalibut', - 'https://thumbs2.redgifs.com/PalegoldenrodRawHalibut.mp4'), + {'https://thumbs2.redgifs.com/PalegoldenrodRawHalibut.mp4'}), + ('https://redgifs.com/watch/hollowintentsnowyowl', + {'https://thumbs2.redgifs.com/HollowIntentSnowyowl-large.jpg'}), + ('https://www.redgifs.com/watch/lustrousstickywaxwing', + {'https://thumbs2.redgifs.com/EntireEnchantingHypsilophodon-large.jpg', + 'https://thumbs2.redgifs.com/FancyMagnificentAdamsstaghornedbeetle-large.jpg', + 'https://thumbs2.redgifs.com/LustrousStickyWaxwing-large.jpg', + 'https://thumbs2.redgifs.com/ParchedWindyArmyworm-large.jpg', + 'https://thumbs2.redgifs.com/ThunderousColorlessErmine-large.jpg', + 'https://thumbs2.redgifs.com/UnripeUnkemptWoodpecker-large.jpg'}), )) -def test_get_link(test_url: str, expected: str): +def test_get_link(test_url: str, expected: set[str]): result = Redgifs._get_link(test_url) assert result == expected @pytest.mark.online -@pytest.mark.parametrize(('test_url', 'expected_hash'), ( - ('https://redgifs.com/watch/frighteningvictorioussalamander', '4007c35d9e1f4b67091b5f12cffda00a'), - ('https://redgifs.com/watch/springgreendecisivetaruca', '8dac487ac49a1f18cc1b4dabe23f0869'), - ('https://redgifs.com/watch/leafysaltydungbeetle', '076792c660b9c024c0471ef4759af8bd'), - ('https://www.redgifs.com/watch/palegoldenrodrawhalibut', '46d5aa77fe80c6407de1ecc92801c10e'), +@pytest.mark.parametrize(('test_url', 'expected_hashes'), ( + ('https://redgifs.com/watch/frighteningvictorioussalamander', {'4007c35d9e1f4b67091b5f12cffda00a'}), + ('https://redgifs.com/watch/springgreendecisivetaruca', {'8dac487ac49a1f18cc1b4dabe23f0869'}), + ('https://redgifs.com/watch/leafysaltydungbeetle', {'076792c660b9c024c0471ef4759af8bd'}), + ('https://www.redgifs.com/watch/palegoldenrodrawhalibut', {'46d5aa77fe80c6407de1ecc92801c10e'}), + ('https://redgifs.com/watch/hollowintentsnowyowl', + {'5ee51fa15e0a58e98f11dea6a6cca771'}), + ('https://www.redgifs.com/watch/lustrousstickywaxwing', + {'b461e55664f07bed8d2f41d8586728fa', + '30ba079a8ed7d7adf17929dc3064c10f', + '0d4f149d170d29fc2f015c1121bab18b', + '53987d99cfd77fd65b5fdade3718f9f1', + 'fb2e7d972846b83bf4016447d3060d60', + '44fb28f72ec9a5cca63fa4369ab4f672'}), )) -def test_download_resource(test_url: str, expected_hash: str): +def test_download_resource(test_url: str, expected_hashes: set[str]): mock_submission = Mock() mock_submission.url = test_url test_site = Redgifs(mock_submission) - resources = test_site.find_resources() - assert len(resources) == 1 - assert isinstance(resources[0], Resource) - resources[0].download() - assert resources[0].hash.hexdigest() == expected_hash + results = test_site.find_resources() + assert all([isinstance(res, Resource) for res in results]) + [res.download() for res in results] + hashes = set([res.hash.hexdigest() for res in results]) + assert hashes == set(expected_hashes) From e8d767050f1be80311b293bc541b3600f52f0e61 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 18 Feb 2022 12:30:38 +1000 Subject: [PATCH 07/30] Add file scheme naming for archiver --- bdfr/__main__.py | 4 ++-- .../integration_tests/test_archive_integration.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/bdfr/__main__.py b/bdfr/__main__.py index 56ffb0f..451a7f0 100644 --- a/bdfr/__main__.py +++ b/bdfr/__main__.py @@ -17,6 +17,8 @@ _common_options = [ click.option('--authenticate', is_flag=True, default=None), click.option('--config', type=str, default=None), click.option('--disable-module', multiple=True, default=None, type=str), + click.option('--file-scheme', default=None, type=str), + click.option('--folder-scheme', default=None, type=str), click.option('--ignore-user', type=str, multiple=True, default=None), click.option('--include-id-file', multiple=True, default=None), click.option('--log', type=str, default=None), @@ -38,8 +40,6 @@ _common_options = [ ] _downloader_options = [ - click.option('--file-scheme', default=None, type=str), - click.option('--folder-scheme', default=None, type=str), click.option('--make-hard-links', is_flag=True, default=None), click.option('--max-wait-time', type=int, default=None), click.option('--no-dupes', is_flag=True, default=None), diff --git a/tests/integration_tests/test_archive_integration.py b/tests/integration_tests/test_archive_integration.py index 5ef04a6..2234c5a 100644 --- a/tests/integration_tests/test_archive_integration.py +++ b/tests/integration_tests/test_archive_integration.py @@ -121,3 +121,18 @@ def test_cli_archive_ignore_user(test_args: list[str], tmp_path: Path): assert result.exit_code == 0 assert 'being an ignored user' in result.output assert 'Attempting to archive submission' not in result.output + + +@pytest.mark.online +@pytest.mark.reddit +@pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests') +@pytest.mark.parametrize('test_args', ( + ['--file-scheme', '{TITLE}', '-l', 'suy011'], +)) +def test_cli_archive_file_format(test_args: list[str], tmp_path: Path): + runner = CliRunner() + test_args = create_basic_args_for_archive_runner(test_args, tmp_path) + result = runner.invoke(cli, test_args) + assert result.exit_code == 0 + assert 'Attempting to archive submission' in result.output + assert re.search('format at /.+?/Judge says Trump and two adult', result.output) From a59916939979f7e540e80190a3d4b2ae0a87267e Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 18 Feb 2022 12:42:42 +1000 Subject: [PATCH 08/30] Increase version number --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 198ebe7..67a1deb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ description_file = README.md description_content_type = text/markdown home_page = https://github.com/aliparlakci/bulk-downloader-for-reddit keywords = reddit, download, archive -version = 2.5.2 +version = 2.6.0 author = Ali Parlakci author_email = parlakciali@gmail.com maintainer = Serene Arc From f49a1d7a2d585f4743c526a745dacc231e439038 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 18 Feb 2022 12:49:46 +1000 Subject: [PATCH 09/30] Fix gfycat after redgifs changes --- bdfr/site_downloaders/gfycat.py | 4 ++-- tests/site_downloaders/test_gfycat.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bdfr/site_downloaders/gfycat.py b/bdfr/site_downloaders/gfycat.py index 6accaab..c8da9df 100644 --- a/bdfr/site_downloaders/gfycat.py +++ b/bdfr/site_downloaders/gfycat.py @@ -21,7 +21,7 @@ class Gfycat(Redgifs): return super().find_resources(authenticator) @staticmethod - def _get_link(url: str) -> str: + def _get_link(url: str) -> set[str]: gfycat_id = re.match(r'.*/(.*?)/?$', url).group(1) url = 'https://gfycat.com/' + gfycat_id @@ -39,4 +39,4 @@ class Gfycat(Redgifs): raise SiteDownloaderError(f'Failed to download Gfycat link {url}: {e}') except json.JSONDecodeError as e: raise SiteDownloaderError(f'Did not receive valid JSON data: {e}') - return out + return {out,} diff --git a/tests/site_downloaders/test_gfycat.py b/tests/site_downloaders/test_gfycat.py index 981d01d..3b40840 100644 --- a/tests/site_downloaders/test_gfycat.py +++ b/tests/site_downloaders/test_gfycat.py @@ -16,7 +16,7 @@ from bdfr.site_downloaders.gfycat import Gfycat )) def test_get_link(test_url: str, expected_url: str): result = Gfycat._get_link(test_url) - assert result == expected_url + assert result.pop() == expected_url @pytest.mark.online From 1abb7768c34a4ff3f8f09ccb25927d8349091f54 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 18 Feb 2022 12:51:51 +1000 Subject: [PATCH 10/30] Update test hashes --- .../fallback_downloaders/test_ytdlp_fallback.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py index 2c4a4f6..503f14e 100644 --- a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py +++ b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py @@ -35,8 +35,8 @@ def test_info_extraction_bad(test_url: str): @pytest.mark.parametrize(('test_url', 'expected_hash'), ( ('https://streamable.com/dt46y', 'b7e465adaade5f2b6d8c2b4b7d0a2878'), ('https://streamable.com/t8sem', '49b2d1220c485455548f1edbc05d4ecf'), - ('https://www.reddit.com/r/specializedtools/comments/n2nw5m/bamboo_splitter/', '21968d3d92161ea5e0abdcaf6311b06c'), - ('https://v.redd.it/9z1dnk3xr5k61', '351a2b57e888df5ccbc508056511f38d'), + ('https://www.reddit.com/r/specializedtools/comments/n2nw5m/bamboo_splitter/', '0a406f3d2e09b3d3ba43bf97185b83e3'), + ('https://v.redd.it/9z1dnk3xr5k61', 'c7765c33972549465c87dcbd59eb3d5d'), )) def test_find_resources(test_url: str, expected_hash: str): test_submission = MagicMock() From 12982c00cdd7bdbd69987cb686118194d0b26f40 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Sun, 20 Feb 2022 15:48:02 +1000 Subject: [PATCH 11/30] Switch redgifs to dynamic file extensions --- bdfr/site_downloaders/redgifs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bdfr/site_downloaders/redgifs.py b/bdfr/site_downloaders/redgifs.py index 12fb24d..f7ea56a 100644 --- a/bdfr/site_downloaders/redgifs.py +++ b/bdfr/site_downloaders/redgifs.py @@ -18,7 +18,7 @@ class Redgifs(BaseDownloader): def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]: media_urls = self._get_link(self.post.url) - return [Resource(self.post, m, Resource.retry_download(m), '.mp4') for m in media_urls] + return [Resource(self.post, m, Resource.retry_download(m), None) for m in media_urls] @staticmethod def _get_link(url: str) -> set[str]: From 2bdeaf26604b7c2e937176cf1139aa348d0fa6f7 Mon Sep 17 00:00:00 2001 From: sinclairkosh <102016413+sinclairkosh@users.noreply.github.com> Date: Tue, 22 Mar 2022 05:53:43 +1100 Subject: [PATCH 12/30] Update Readme with some command clarifications Clarify that fact that downloading by user doesn't work the same way as downloading by subreddit. Feel free to user a better example username. :) --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index b84aa3d..47c0dde 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,12 @@ However, these commands are not enough. You should chain parameters in [Options] python3 -m bdfr download ./path/to/output --subreddit Python -L 10 ``` ```bash +python3 -m bdfr download ./path/to/output --user reddituser --submitted -L 100 +``` +```bash +python3 -m bdfr download ./path/to/output --user reddituser --submitted --all-comments --comment-context +``` +```bash python3 -m bdfr download ./path/to/output --user me --saved --authenticate -L 25 --file-scheme '{POSTID}' ``` ```bash From 9f3dcece4d2af458b819bd68e26d3adbea9ffb28 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 25 Mar 2022 10:50:52 +1000 Subject: [PATCH 13/30] Strip any newline characters from names --- bdfr/file_name_formatter.py | 3 +++ tests/test_file_name_formatter.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bdfr/file_name_formatter.py b/bdfr/file_name_formatter.py index 3e8832b..1dabd34 100644 --- a/bdfr/file_name_formatter.py +++ b/bdfr/file_name_formatter.py @@ -111,6 +111,9 @@ class FileNameFormatter: if not resource.extension: raise BulkDownloaderException(f'Resource from {resource.url} has no extension') file_name = str(self._format_name(resource.source_submission, self.file_format_string)) + + file_name = re.sub(r'\n', ' ', file_name) + if not re.match(r'.*\.$', file_name) and not re.match(r'^\..*', resource.extension): ending = index + '.' + resource.extension else: diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index 30fac77..bd9d058 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -16,6 +16,7 @@ from bdfr.file_name_formatter import FileNameFormatter from bdfr.resource import Resource from bdfr.site_downloaders.base_downloader import BaseDownloader from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallback +from bdfr.site_downloaders.self_post import SelfPost @pytest.fixture() @@ -406,6 +407,7 @@ def test_windows_max_path(tmp_path: Path): @pytest.mark.parametrize(('test_reddit_id', 'test_downloader', 'expected_names'), ( ('gphmnr', YtdlpFallback, {'He has a lot to say today.mp4'}), ('d0oir2', YtdlpFallback, {"Crunk's finest moment. Welcome to the new subreddit!.mp4"}), + ('jiecu', SelfPost, {'Reston, VA Some info regarding shelters in the area..txt'}) )) def test_name_submission( test_reddit_id: str, @@ -418,4 +420,4 @@ def test_name_submission( test_formatter = FileNameFormatter('{TITLE}', '', '') results = test_formatter.format_resource_paths(test_resources, Path('.')) results = set([r[0].name for r in results]) - assert expected_names == results + assert results == expected_names From 53d7ce2e5d9ec313fde62e184b6b356340c0a0d9 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 25 Mar 2022 10:52:49 +1000 Subject: [PATCH 14/30] Add second test case --- tests/test_file_name_formatter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index bd9d058..c9e049e 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -407,7 +407,8 @@ def test_windows_max_path(tmp_path: Path): @pytest.mark.parametrize(('test_reddit_id', 'test_downloader', 'expected_names'), ( ('gphmnr', YtdlpFallback, {'He has a lot to say today.mp4'}), ('d0oir2', YtdlpFallback, {"Crunk's finest moment. Welcome to the new subreddit!.mp4"}), - ('jiecu', SelfPost, {'Reston, VA Some info regarding shelters in the area..txt'}) + ('jiecu', SelfPost, {'Reston, VA Some info regarding shelters in the area..txt'}), + ('gui1i', SelfPost, {'The "Beer and Ear offer for those who need help in the \'burbs of North Dallas....txt'}), )) def test_name_submission( test_reddit_id: str, From e068c9ce563de95c5ae8a6d4f8b758288be72dce Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Mon, 18 Apr 2022 16:03:24 -0500 Subject: [PATCH 15/30] readme: make --search info a bit more clear --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 47c0dde..28c0b8a 100644 --- a/README.md +++ b/README.md @@ -98,8 +98,8 @@ The following options are common between both the `archive` and `download` comma - This option will make the BDFR use the supplied user's saved posts list as a download source - This requires an authenticated Reddit instance, using the `--authenticate` flag, as well as `--user` set to `me` - `--search` - - This will apply the specified search term to specific lists when scraping submissions - - A search term can only be applied to subreddits and multireddits, supplied with the `- s` and `-m` flags respectively + - This will apply the input search term to specific lists when scraping submissions + - A search term can only be applied when using the `--subreddit` and `--multireddit` flags - `--submitted` - This will use a user's submissions as a source - A user must be specified with `--user` From ad172841e2f52fe5c8b98af57ecdb1f607ac7fe2 Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Mon, 18 Apr 2022 00:30:17 -0500 Subject: [PATCH 16/30] Use stdout --- scripts/extract_failed_ids.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/extract_failed_ids.sh b/scripts/extract_failed_ids.sh index 8addf7e..64d1e72 100755 --- a/scripts/extract_failed_ids.sh +++ b/scripts/extract_failed_ids.sh @@ -7,17 +7,10 @@ else exit 1 fi -if [ -n "$2" ]; then - output="$2" - echo "Outputting IDs to $output" -else - output="./failed.txt" -fi - { grep 'Could not download submission' "$file" | awk '{ print $12 }' | rev | cut -c 2- | rev ; grep 'Failed to download resource' "$file" | awk '{ print $15 }' ; grep 'failed to download submission' "$file" | awk '{ print $14 }' | rev | cut -c 2- | rev ; grep 'Failed to write file' "$file" | awk '{ print $14 }' ; grep 'skipped due to disabled module' "$file" | awk '{ print $9 }' ; -} >>"$output" +} From e4a44f1e252f684455a97447f17322d1f34cae72 Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Mon, 18 Apr 2022 20:42:50 -0500 Subject: [PATCH 17/30] okay --- scripts/tests/test_extract_failed_ids.bats | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/tests/test_extract_failed_ids.bats b/scripts/tests/test_extract_failed_ids.bats index 75b9bff..a716cba 100644 --- a/scripts/tests/test_extract_failed_ids.bats +++ b/scripts/tests/test_extract_failed_ids.bats @@ -13,31 +13,31 @@ teardown() { } @test "fail no downloader module" { - run ../extract_failed_ids.sh ./example_logfiles/failed_no_downloader.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_no_downloader.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "3" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } @test "fail resource error" { - run ../extract_failed_ids.sh ./example_logfiles/failed_resource_error.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_resource_error.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } @test "fail site downloader error" { - run ../extract_failed_ids.sh ./example_logfiles/failed_sitedownloader_error.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_sitedownloader_error.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "2" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } @test "fail failed file write" { - run ../extract_failed_ids.sh ./example_logfiles/failed_write_error.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_write_error.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } @test "fail disabled module" { - run ../extract_failed_ids.sh ./example_logfiles/failed_disabled_module.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_disabled_module.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } From eb8f9d5876437ce28be8be184895874ebef14e3c Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Mon, 18 Apr 2022 20:47:35 -0500 Subject: [PATCH 18/30] oh there's another one --- scripts/extract_successful_ids.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/extract_successful_ids.sh b/scripts/extract_successful_ids.sh index 011ba6c..e8f482e 100755 --- a/scripts/extract_successful_ids.sh +++ b/scripts/extract_successful_ids.sh @@ -7,17 +7,10 @@ else exit 1 fi -if [ -n "$2" ]; then - output="$2" - echo "Outputting IDs to $output" -else - output="./successful.txt" -fi - { grep 'Downloaded submission' "$file" | awk '{ print $(NF-2) }' ; grep 'Resource hash' "$file" | awk '{ print $(NF-2) }' ; grep 'Download filter' "$file" | awk '{ print $(NF-3) }' ; grep 'already exists, continuing' "$file" | awk '{ print $(NF-3) }' ; grep 'Hard link made' "$file" | awk '{ print $(NF) }' ; -} >> "$output" +} From efea01e56f112e1c46645813296371c3c6e020a0 Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Mon, 18 Apr 2022 20:48:02 -0500 Subject: [PATCH 19/30] and this one too --- scripts/tests/test_extract_successful_ids.bats | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/tests/test_extract_successful_ids.bats b/scripts/tests/test_extract_successful_ids.bats index 364bedb..caa8dd1 100644 --- a/scripts/tests/test_extract_successful_ids.bats +++ b/scripts/tests/test_extract_successful_ids.bats @@ -8,31 +8,31 @@ teardown() { } @test "success downloaded submission" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_downloaded_submission.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_downloaded_submission.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "7" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; } @test "success resource hash" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_resource_hash.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_resource_hash.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; } @test "success download filter" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_download_filter.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_download_filter.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "3" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; } @test "success already exists" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_already_exists.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_already_exists.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "3" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; } @test "success hard link" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_hard_link.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_hard_link.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; } From decb13b5db45624ff7aaf24e004a56445a7cfcc8 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Sun, 8 May 2022 12:18:31 +1000 Subject: [PATCH 20/30] Replace old Vidble test cases --- tests/site_downloaders/test_vidble.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/site_downloaders/test_vidble.py b/tests/site_downloaders/test_vidble.py index 0c5ebb2..50ca808 100644 --- a/tests/site_downloaders/test_vidble.py +++ b/tests/site_downloaders/test_vidble.py @@ -30,8 +30,8 @@ def test_change_med_url(test_url: str, expected: str): 'https://www.vidble.com/VWuNsnLJMD.jpg', 'https://www.vidble.com/sMmM8O650W.jpg', }), - ('https://vidble.com/watch?v=0q4nWakqM6kzQWxlePD8N62Dsflev0N9', { - 'https://www.vidble.com/0q4nWakqM6kzQWxlePD8N62Dsflev0N9.mp4', + ('https://www.vidble.com/watch?v=joC6b7cgs2Tnucx7dhDoyqKPbr7TQUA5', { + 'https://www.vidble.com/joC6b7cgs2Tnucx7dhDoyqKPbr7TQUA5.mp4', }), ('https://www.vidble.com/pHuwWkOcEb', { 'https://www.vidble.com/pHuwWkOcEb.jpg', @@ -55,8 +55,8 @@ def test_get_links(test_url: str, expected: set[str]): 'b31a942cd8cdda218ed547bbc04c3a27', '6f77c570b451eef4222804bd52267481', }), - ('https://vidble.com/watch?v=0q4nWakqM6kzQWxlePD8N62Dsflev0N9', { - 'cebe9d5f24dba3b0443e5097f160ca83', + ('https://www.vidble.com/watch?v=joC6b7cgs2Tnucx7dhDoyqKPbr7TQUA5', { + 'ec5f7a7f74a4dd55c740cbfd4d3bf9ab', }), ('https://www.vidble.com/pHuwWkOcEb', { '585f486dd0b2f23a57bddbd5bf185bc7', From 7d4916919d15aef08ca9b545ac65689f0f1b08fb Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Wed, 6 Jul 2022 15:04:05 +1000 Subject: [PATCH 21/30] Add test case --- .../site_downloaders/fallback_downloaders/test_ytdlp_fallback.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py index 503f14e..c600f6f 100644 --- a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py +++ b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py @@ -15,6 +15,7 @@ from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallb ('https://www.youtube.com/watch?v=P19nvJOmqCc', True), ('https://www.example.com/test', False), ('https://milesmatrix.bandcamp.com/album/la-boum/', False), + ('https://v.redd.it/54i8fvzev3u81', True), )) def test_can_handle_link(test_url: str, expected: bool): result = YtdlpFallback.can_handle_link(test_url) From 2d365b612b28c1efb637dda95a73266d10633170 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Wed, 6 Jul 2022 16:50:02 +1000 Subject: [PATCH 22/30] Fix some test cases --- tests/integration_tests/test_download_integration.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py index 75216dd..ffae0d4 100644 --- a/tests/integration_tests/test_download_integration.py +++ b/tests/integration_tests/test_download_integration.py @@ -97,7 +97,6 @@ def test_cli_download_user_specific_subreddits(test_args: list[str], tmp_path: P ['-l', 'm2601g'], ['-l', 'https://www.reddit.com/r/TrollXChromosomes/comments/m2601g/its_a_step_in_the_right_direction/'], ['-l', 'm3hxzd'], # Really long title used to overflow filename limit - ['-l', 'm3kua3'], # Has a deleted user ['-l', 'm5bqkf'], # Resource leading to a 404 )) def test_cli_download_links(test_args: list[str], tmp_path: Path): @@ -313,9 +312,8 @@ def test_cli_download_file_scheme_warning(test_args: list[str], tmp_path: Path): @pytest.mark.reddit @pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests') @pytest.mark.parametrize('test_args', ( - ['-l', 'm2601g', '--disable-module', 'Direct'], - ['-l', 'nnb9vs', '--disable-module', 'YoutubeDlFallback'], - ['-l', 'nnb9vs', '--disable-module', 'youtubedlfallback'], + ['-l', 'm2601g', '--disable-module', 'SelfPost'], + ['-l', 'nnb9vs', '--disable-module', 'YtdlpFallback'], )) def test_cli_download_disable_modules(test_args: list[str], tmp_path: Path): runner = CliRunner() From 8c59329ffab9ecf977fc978650350158a054ea2f Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Wed, 6 Jul 2022 16:52:01 +1000 Subject: [PATCH 23/30] Add exclusion options to archiver --- bdfr/__main__.py | 4 ++-- bdfr/archiver.py | 3 +++ bdfr/configuration.py | 2 +- .../integration_tests/test_archive_integration.py | 15 +++++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/bdfr/__main__.py b/bdfr/__main__.py index 451a7f0..45450ed 100644 --- a/bdfr/__main__.py +++ b/bdfr/__main__.py @@ -17,6 +17,8 @@ _common_options = [ click.option('--authenticate', is_flag=True, default=None), click.option('--config', type=str, default=None), click.option('--disable-module', multiple=True, default=None, type=str), + click.option('--exclude-id', default=None, multiple=True), + click.option('--exclude-id-file', default=None, multiple=True), click.option('--file-scheme', default=None, type=str), click.option('--folder-scheme', default=None, type=str), click.option('--ignore-user', type=str, multiple=True, default=None), @@ -44,8 +46,6 @@ _downloader_options = [ click.option('--max-wait-time', type=int, default=None), click.option('--no-dupes', is_flag=True, default=None), click.option('--search-existing', is_flag=True, default=None), - click.option('--exclude-id', default=None, multiple=True), - click.option('--exclude-id-file', default=None, multiple=True), click.option('--skip', default=None, multiple=True), click.option('--skip-domain', default=None, multiple=True), click.option('--skip-subreddit', default=None, multiple=True), diff --git a/bdfr/archiver.py b/bdfr/archiver.py index 559dcc1..214111f 100644 --- a/bdfr/archiver.py +++ b/bdfr/archiver.py @@ -34,6 +34,9 @@ class Archiver(RedditConnector): f'Submission {submission.id} in {submission.subreddit.display_name} skipped' f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user') continue + if submission.id in self.excluded_submission_ids: + logger.debug(f'Object {submission.id} in exclusion list, skipping') + continue logger.debug(f'Attempting to archive submission {submission.id}') self.write_entry(submission) diff --git a/bdfr/configuration.py b/bdfr/configuration.py index ef24e36..c8fb323 100644 --- a/bdfr/configuration.py +++ b/bdfr/configuration.py @@ -35,7 +35,7 @@ class Configuration(Namespace): self.skip_subreddit: list[str] = [] self.sort: str = 'hot' self.submitted: bool = False - self.subscribed: bool = True + self.subscribed: bool = False self.subreddit: list[str] = [] self.time: str = 'all' self.time_format = None diff --git a/tests/integration_tests/test_archive_integration.py b/tests/integration_tests/test_archive_integration.py index 2234c5a..744e343 100644 --- a/tests/integration_tests/test_archive_integration.py +++ b/tests/integration_tests/test_archive_integration.py @@ -136,3 +136,18 @@ def test_cli_archive_file_format(test_args: list[str], tmp_path: Path): assert result.exit_code == 0 assert 'Attempting to archive submission' in result.output assert re.search('format at /.+?/Judge says Trump and two adult', result.output) + + +@pytest.mark.online +@pytest.mark.reddit +@pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests') +@pytest.mark.parametrize('test_args', ( + ['-l', 'm2601g', '--exclude-id', 'm2601g'], +)) +def test_cli_archive_links_exclusion(test_args: list[str], tmp_path: Path): + runner = CliRunner() + test_args = create_basic_args_for_archive_runner(test_args, tmp_path) + result = runner.invoke(cli, test_args) + assert result.exit_code == 0 + assert 'in exclusion list' in result.output + assert 'Attempting to archive' not in result.output From 3fd5bad4070c35ce60c35ab73229108e06f5afdb Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Thu, 7 Jul 2022 11:39:42 +1000 Subject: [PATCH 24/30] Update some test hashes --- .../fallback_downloaders/test_ytdlp_fallback.py | 4 ++-- tests/site_downloaders/test_youtube.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py index c600f6f..23c95f9 100644 --- a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py +++ b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py @@ -36,8 +36,8 @@ def test_info_extraction_bad(test_url: str): @pytest.mark.parametrize(('test_url', 'expected_hash'), ( ('https://streamable.com/dt46y', 'b7e465adaade5f2b6d8c2b4b7d0a2878'), ('https://streamable.com/t8sem', '49b2d1220c485455548f1edbc05d4ecf'), - ('https://www.reddit.com/r/specializedtools/comments/n2nw5m/bamboo_splitter/', '0a406f3d2e09b3d3ba43bf97185b83e3'), - ('https://v.redd.it/9z1dnk3xr5k61', 'c7765c33972549465c87dcbd59eb3d5d'), + ('https://www.reddit.com/r/specializedtools/comments/n2nw5m/bamboo_splitter/', '49316899440ea1c3b74d5640d9d527c1'), + ('https://v.redd.it/9z1dnk3xr5k61', '76d5e6d7f4f9e1910c6c22b54dfa804f'), )) def test_find_resources(test_url: str, expected_hash: str): test_submission = MagicMock() diff --git a/tests/site_downloaders/test_youtube.py b/tests/site_downloaders/test_youtube.py index 684eb20..ce1abb8 100644 --- a/tests/site_downloaders/test_youtube.py +++ b/tests/site_downloaders/test_youtube.py @@ -15,7 +15,7 @@ from bdfr.site_downloaders.youtube import Youtube @pytest.mark.parametrize(('test_url', 'expected_hash'), ( ('https://www.youtube.com/watch?v=uSm2VDgRIUs', '2d60b54582df5b95ec72bb00b580d2ff'), ('https://www.youtube.com/watch?v=GcI7nxQj7HA', '5db0fc92a0a7fb9ac91e63505eea9cf0'), - ('https://youtu.be/TMqPOlp4tNo', 'f68c00b018162857f3df4844c45302e7'), # Age restricted + ('https://youtu.be/TMqPOlp4tNo', 'ceb4c2cb1a9bf79617623b2aa57e18fd'), # Age restricted )) def test_find_resources_good(test_url: str, expected_hash: str): test_submission = MagicMock() From 7315afeafd93f91934fa381dba8dcc1653f6fe1b Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Thu, 7 Jul 2022 12:07:53 +1000 Subject: [PATCH 25/30] Update test parameter --- tests/test_file_name_formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index c9e049e..9c0a8bb 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -407,7 +407,7 @@ def test_windows_max_path(tmp_path: Path): @pytest.mark.parametrize(('test_reddit_id', 'test_downloader', 'expected_names'), ( ('gphmnr', YtdlpFallback, {'He has a lot to say today.mp4'}), ('d0oir2', YtdlpFallback, {"Crunk's finest moment. Welcome to the new subreddit!.mp4"}), - ('jiecu', SelfPost, {'Reston, VA Some info regarding shelters in the area..txt'}), + ('jiecu', SelfPost, {'[deleted by user].txt'}), ('gui1i', SelfPost, {'The "Beer and Ear offer for those who need help in the \'burbs of North Dallas....txt'}), )) def test_name_submission( From 4f876eecbc23360d6d6555787efc528a02dc9ad8 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 15 Jul 2022 14:22:55 +1000 Subject: [PATCH 26/30] Remove bugged test case --- tests/archive_entry/test_submission_archive_entry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/archive_entry/test_submission_archive_entry.py b/tests/archive_entry/test_submission_archive_entry.py index 60f47b5..045eabd 100644 --- a/tests/archive_entry/test_submission_archive_entry.py +++ b/tests/archive_entry/test_submission_archive_entry.py @@ -34,7 +34,7 @@ def test_get_comments(test_submission_id: str, min_comments: int, reddit_instanc 'created_utc': 1615583837, 'permalink': '/r/australia/comments/m3reby/this_little_guy_fell_out_of_a_tree_and_in_front/' }), - ('m3kua3', {'author': 'DELETED'}), + # TODO: add deleted user test case )) def test_get_post_details(test_submission_id: str, expected_dict: dict, reddit_instance: praw.Reddit): test_submission = reddit_instance.submission(id=test_submission_id) From 7d4eb47643f5f357e6361e05cd76464f992410aa Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 15 Jul 2022 15:05:07 +1000 Subject: [PATCH 27/30] Rename class --- bdfr/site_downloaders/download_factory.py | 4 ++-- bdfr/site_downloaders/vreddit.py | 2 +- tests/site_downloaders/test_download_factory.py | 6 +++--- tests/site_downloaders/test_vreddit.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py index f6f4895..96e9a42 100644 --- a/bdfr/site_downloaders/download_factory.py +++ b/bdfr/site_downloaders/download_factory.py @@ -17,8 +17,8 @@ from bdfr.site_downloaders.pornhub import PornHub from bdfr.site_downloaders.redgifs import Redgifs from bdfr.site_downloaders.self_post import SelfPost from bdfr.site_downloaders.vidble import Vidble +from bdfr.site_downloaders.vreddit import VReddit from bdfr.site_downloaders.youtube import Youtube -from bdfr.site_downloaders.vreddit import Vreddit class DownloadFactory: @@ -49,7 +49,7 @@ class DownloadFactory: elif re.match(r'i\.redd\.it.*', sanitised_url): return Direct elif re.match(r'v\.redd\.it.*', sanitised_url): - return Vreddit + return VReddit elif re.match(r'pornhub\.com.*', sanitised_url): return PornHub elif re.match(r'vidble\.com', sanitised_url): diff --git a/bdfr/site_downloaders/vreddit.py b/bdfr/site_downloaders/vreddit.py index b1117a0..fda6dac 100644 --- a/bdfr/site_downloaders/vreddit.py +++ b/bdfr/site_downloaders/vreddit.py @@ -16,7 +16,7 @@ from bdfr.site_downloaders.base_downloader import BaseDownloader logger = logging.getLogger(__name__) -class Vreddit(BaseDownloader): +class VReddit(BaseDownloader): def __init__(self, post: Submission): super().__init__(post) diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py index 30e19f1..dcb5303 100644 --- a/tests/site_downloaders/test_download_factory.py +++ b/tests/site_downloaders/test_download_factory.py @@ -16,8 +16,8 @@ from bdfr.site_downloaders.imgur import Imgur from bdfr.site_downloaders.pornhub import PornHub from bdfr.site_downloaders.redgifs import Redgifs from bdfr.site_downloaders.self_post import SelfPost +from bdfr.site_downloaders.vreddit import VReddit from bdfr.site_downloaders.youtube import Youtube -from bdfr.site_downloaders.vreddit import Vreddit @pytest.mark.online @@ -43,13 +43,13 @@ from bdfr.site_downloaders.vreddit import Vreddit ('https://i.imgur.com/3SKrQfK.jpg?1', Direct), ('https://dynasty-scans.com/system/images_images/000/017/819/original/80215103_p0.png?1612232781', Direct), ('https://m.imgur.com/a/py3RW0j', Imgur), - ('https://v.redd.it/9z1dnk3xr5k61', Vreddit), + ('https://v.redd.it/9z1dnk3xr5k61', VReddit), ('https://streamable.com/dt46y', YtdlpFallback), ('https://vimeo.com/channels/31259/53576664', YtdlpFallback), ('http://video.pbs.org/viralplayer/2365173446/', YtdlpFallback), ('https://www.pornhub.com/view_video.php?viewkey=ph5a2ee0461a8d0', PornHub), ('https://www.patreon.com/posts/minecart-track-59346560', Gallery), - ('https://v.redd.it/9z1dnk3xr5k61', Vreddit) + ('https://v.redd.it/9z1dnk3xr5k61', VReddit) )) def test_factory_lever_good(test_submission_url: str, expected_class: BaseDownloader, reddit_instance: praw.Reddit): result = DownloadFactory.pull_lever(test_submission_url) diff --git a/tests/site_downloaders/test_vreddit.py b/tests/site_downloaders/test_vreddit.py index 65428b5..f1f8219 100644 --- a/tests/site_downloaders/test_vreddit.py +++ b/tests/site_downloaders/test_vreddit.py @@ -7,7 +7,7 @@ import pytest from bdfr.exceptions import NotADownloadableLinkError from bdfr.resource import Resource -from bdfr.site_downloaders.vreddit import Vreddit +from bdfr.site_downloaders.vreddit import VReddit @pytest.mark.online @@ -18,7 +18,7 @@ from bdfr.site_downloaders.vreddit import Vreddit def test_find_resources_good(test_url: str, expected_hash: str): test_submission = MagicMock() test_submission.url = test_url - downloader = Vreddit(test_submission) + downloader = VReddit(test_submission) resources = downloader.find_resources() assert len(resources) == 1 assert isinstance(resources[0], Resource) @@ -34,6 +34,6 @@ def test_find_resources_good(test_url: str, expected_hash: str): def test_find_resources_bad(test_url: str): test_submission = MagicMock() test_submission.url = test_url - downloader = Vreddit(test_submission) + downloader = VReddit(test_submission) with pytest.raises(NotADownloadableLinkError): downloader.find_resources() From 92779033087e2e0f6674829a78a97d10e0cdaaaf Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 15 Jul 2022 15:12:39 +1000 Subject: [PATCH 28/30] Base VReddit class off of Youtube class --- bdfr/site_downloaders/vreddit.py | 41 ++++---------------------------- bdfr/site_downloaders/youtube.py | 7 +++++- 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/bdfr/site_downloaders/vreddit.py b/bdfr/site_downloaders/vreddit.py index fda6dac..ad526b4 100644 --- a/bdfr/site_downloaders/vreddit.py +++ b/bdfr/site_downloaders/vreddit.py @@ -11,12 +11,12 @@ from praw.models import Submission from bdfr.exceptions import NotADownloadableLinkError, SiteDownloaderError from bdfr.resource import Resource from bdfr.site_authenticator import SiteAuthenticator -from bdfr.site_downloaders.base_downloader import BaseDownloader +from bdfr.site_downloaders.youtube import Youtube logger = logging.getLogger(__name__) -class VReddit(BaseDownloader): +class VReddit(Youtube): def __init__(self, post: Submission): super().__init__(post) @@ -30,47 +30,14 @@ class VReddit(BaseDownloader): res = Resource(self.post, self.post.url, download_function, extension) return [res] - def _download_video(self, ytdl_options: dict) -> Callable: - yt_logger = logging.getLogger('youtube-dl') - yt_logger.setLevel(logging.CRITICAL) - ytdl_options['quiet'] = True - ytdl_options['logger'] = yt_logger - - def download(_: dict) -> bytes: - with tempfile.TemporaryDirectory() as temp_dir: - download_path = Path(temp_dir).resolve() - ytdl_options['outtmpl'] = str(download_path) + '/' + 'test.%(ext)s' - try: - with yt_dlp.YoutubeDL(ytdl_options) as ydl: - ydl.download([self.post.url]) - except yt_dlp.DownloadError as e: - raise SiteDownloaderError(f'Vreddit download failed: {e}') - - downloaded_files = list(download_path.iterdir()) - if len(downloaded_files) > 0: - downloaded_file = downloaded_files[0] - else: - raise NotADownloadableLinkError(f"No media exists in the URL {self.post.url}") - with open(downloaded_file, 'rb') as file: - content = file.read() - return content - return download - @staticmethod def get_video_attributes(url: str) -> dict: - yt_logger = logging.getLogger('youtube-dl') - yt_logger.setLevel(logging.CRITICAL) - with yt_dlp.YoutubeDL({'logger': yt_logger, }) as ydl: - try: - result = ydl.extract_info(url, download=False) - except Exception as e: - logger.exception(e) - raise NotADownloadableLinkError(f'Video info extraction failed for {url}') + result = VReddit.get_video_data(url) if 'ext' in result: return result else: try: - result = (result["entries"][0]) + result = result["entries"][0] return result except Exception as e: logger.exception(e) diff --git a/bdfr/site_downloaders/youtube.py b/bdfr/site_downloaders/youtube.py index f18f405..70c35ae 100644 --- a/bdfr/site_downloaders/youtube.py +++ b/bdfr/site_downloaders/youtube.py @@ -58,7 +58,7 @@ class Youtube(BaseDownloader): return download @staticmethod - def get_video_attributes(url: str) -> dict: + def get_video_data(url: str) -> dict: yt_logger = logging.getLogger('youtube-dl') yt_logger.setLevel(logging.CRITICAL) with yt_dlp.YoutubeDL({'logger': yt_logger, }) as ydl: @@ -67,6 +67,11 @@ class Youtube(BaseDownloader): except Exception as e: logger.exception(e) raise NotADownloadableLinkError(f'Video info extraction failed for {url}') + return result + + @staticmethod + def get_video_attributes(url: str) -> dict: + result = Youtube.get_video_data(url) if 'ext' in result: return result else: From 86e451d49e7e48a89a30b6209174a0a3c435124e Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 15 Jul 2022 15:18:14 +1000 Subject: [PATCH 29/30] Fix test case --- .../fallback_downloaders/test_ytdlp_fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py index 23c95f9..9aeca98 100644 --- a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py +++ b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py @@ -15,7 +15,7 @@ from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallb ('https://www.youtube.com/watch?v=P19nvJOmqCc', True), ('https://www.example.com/test', False), ('https://milesmatrix.bandcamp.com/album/la-boum/', False), - ('https://v.redd.it/54i8fvzev3u81', True), + ('https://v.redd.it/54i8fvzev3u81', False), )) def test_can_handle_link(test_url: str, expected: bool): result = YtdlpFallback.can_handle_link(test_url) From febad9c06c3b8b40e0cbb765a7559593ba5e2632 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 15 Jul 2022 15:47:49 +1000 Subject: [PATCH 30/30] Remove bad test case --- tests/test_file_name_formatter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index 9c0a8bb..21cb8a6 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -408,7 +408,6 @@ def test_windows_max_path(tmp_path: Path): ('gphmnr', YtdlpFallback, {'He has a lot to say today.mp4'}), ('d0oir2', YtdlpFallback, {"Crunk's finest moment. Welcome to the new subreddit!.mp4"}), ('jiecu', SelfPost, {'[deleted by user].txt'}), - ('gui1i', SelfPost, {'The "Beer and Ear offer for those who need help in the \'burbs of North Dallas....txt'}), )) def test_name_submission( test_reddit_id: str,