From 9deef63fdda3bb2623fd035c93aad6ec8c8ef8c2 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 18 Feb 2022 12:04:37 +1000 Subject: [PATCH] Add support for Redgifs images and galleries --- bdfr/site_downloaders/redgifs.py | 31 +++++++++++++---- tests/site_downloaders/test_redgifs.py | 48 ++++++++++++++++++-------- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/bdfr/site_downloaders/redgifs.py b/bdfr/site_downloaders/redgifs.py index a62fedb..12fb24d 100644 --- a/bdfr/site_downloaders/redgifs.py +++ b/bdfr/site_downloaders/redgifs.py @@ -17,11 +17,11 @@ class Redgifs(BaseDownloader): super().__init__(post) def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]: - media_url = self._get_link(self.post.url) - return [Resource(self.post, media_url, Resource.retry_download(media_url), '.mp4')] + media_urls = self._get_link(self.post.url) + return [Resource(self.post, m, Resource.retry_download(m), '.mp4') for m in media_urls] @staticmethod - def _get_link(url: str) -> str: + def _get_link(url: str) -> set[str]: try: redgif_id = re.match(r'.*/(.*?)/?$', url).group(1) except AttributeError: @@ -32,16 +32,33 @@ class Redgifs(BaseDownloader): 'Chrome/90.0.4430.93 Safari/537.36', } - content = Redgifs.retrieve_url(f'https://api.redgifs.com/v1/gfycats/{redgif_id}', headers=headers) + content = Redgifs.retrieve_url(f'https://api.redgifs.com/v2/gifs/{redgif_id}', headers=headers) if content is None: raise SiteDownloaderError('Could not read the page source') try: - out = json.loads(content.text)['gfyItem']['mp4Url'] - except (KeyError, AttributeError): - raise SiteDownloaderError('Failed to find JSON data in page') + response_json = json.loads(content.text) except json.JSONDecodeError as e: raise SiteDownloaderError(f'Received data was not valid JSON: {e}') + out = set() + try: + if response_json['gif']['type'] == 1: # type 1 is a video + out.add(response_json['gif']['urls']['hd']) + elif response_json['gif']['type'] == 2: # type 2 is an image + if response_json['gif']['gallery']: + content = Redgifs.retrieve_url( + f'https://api.redgifs.com/v2/gallery/{response_json["gif"]["gallery"]}', + headers=headers, + ) + response_json = json.loads(content.text) + out = {p['urls']['hd'] for p in response_json['gifs']} + else: + out.add(response_json['gif']['urls']['hd']) + else: + raise KeyError + except (KeyError, AttributeError): + raise SiteDownloaderError('Failed to find JSON data in page') + return out diff --git a/tests/site_downloaders/test_redgifs.py b/tests/site_downloaders/test_redgifs.py index 571f044..a1f571e 100644 --- a/tests/site_downloaders/test_redgifs.py +++ b/tests/site_downloaders/test_redgifs.py @@ -12,30 +12,48 @@ from bdfr.site_downloaders.redgifs import Redgifs @pytest.mark.online @pytest.mark.parametrize(('test_url', 'expected'), ( ('https://redgifs.com/watch/frighteningvictorioussalamander', - 'https://thumbs2.redgifs.com/FrighteningVictoriousSalamander.mp4'), + {'https://thumbs2.redgifs.com/FrighteningVictoriousSalamander.mp4'}), ('https://redgifs.com/watch/springgreendecisivetaruca', - 'https://thumbs2.redgifs.com/SpringgreenDecisiveTaruca.mp4'), + {'https://thumbs2.redgifs.com/SpringgreenDecisiveTaruca.mp4'}), ('https://www.redgifs.com/watch/palegoldenrodrawhalibut', - 'https://thumbs2.redgifs.com/PalegoldenrodRawHalibut.mp4'), + {'https://thumbs2.redgifs.com/PalegoldenrodRawHalibut.mp4'}), + ('https://redgifs.com/watch/hollowintentsnowyowl', + {'https://thumbs2.redgifs.com/HollowIntentSnowyowl-large.jpg'}), + ('https://www.redgifs.com/watch/lustrousstickywaxwing', + {'https://thumbs2.redgifs.com/EntireEnchantingHypsilophodon-large.jpg', + 'https://thumbs2.redgifs.com/FancyMagnificentAdamsstaghornedbeetle-large.jpg', + 'https://thumbs2.redgifs.com/LustrousStickyWaxwing-large.jpg', + 'https://thumbs2.redgifs.com/ParchedWindyArmyworm-large.jpg', + 'https://thumbs2.redgifs.com/ThunderousColorlessErmine-large.jpg', + 'https://thumbs2.redgifs.com/UnripeUnkemptWoodpecker-large.jpg'}), )) -def test_get_link(test_url: str, expected: str): +def test_get_link(test_url: str, expected: set[str]): result = Redgifs._get_link(test_url) assert result == expected @pytest.mark.online -@pytest.mark.parametrize(('test_url', 'expected_hash'), ( - ('https://redgifs.com/watch/frighteningvictorioussalamander', '4007c35d9e1f4b67091b5f12cffda00a'), - ('https://redgifs.com/watch/springgreendecisivetaruca', '8dac487ac49a1f18cc1b4dabe23f0869'), - ('https://redgifs.com/watch/leafysaltydungbeetle', '076792c660b9c024c0471ef4759af8bd'), - ('https://www.redgifs.com/watch/palegoldenrodrawhalibut', '46d5aa77fe80c6407de1ecc92801c10e'), +@pytest.mark.parametrize(('test_url', 'expected_hashes'), ( + ('https://redgifs.com/watch/frighteningvictorioussalamander', {'4007c35d9e1f4b67091b5f12cffda00a'}), + ('https://redgifs.com/watch/springgreendecisivetaruca', {'8dac487ac49a1f18cc1b4dabe23f0869'}), + ('https://redgifs.com/watch/leafysaltydungbeetle', {'076792c660b9c024c0471ef4759af8bd'}), + ('https://www.redgifs.com/watch/palegoldenrodrawhalibut', {'46d5aa77fe80c6407de1ecc92801c10e'}), + ('https://redgifs.com/watch/hollowintentsnowyowl', + {'5ee51fa15e0a58e98f11dea6a6cca771'}), + ('https://www.redgifs.com/watch/lustrousstickywaxwing', + {'b461e55664f07bed8d2f41d8586728fa', + '30ba079a8ed7d7adf17929dc3064c10f', + '0d4f149d170d29fc2f015c1121bab18b', + '53987d99cfd77fd65b5fdade3718f9f1', + 'fb2e7d972846b83bf4016447d3060d60', + '44fb28f72ec9a5cca63fa4369ab4f672'}), )) -def test_download_resource(test_url: str, expected_hash: str): +def test_download_resource(test_url: str, expected_hashes: set[str]): mock_submission = Mock() mock_submission.url = test_url test_site = Redgifs(mock_submission) - resources = test_site.find_resources() - assert len(resources) == 1 - assert isinstance(resources[0], Resource) - resources[0].download() - assert resources[0].hash.hexdigest() == expected_hash + results = test_site.find_resources() + assert all([isinstance(res, Resource) for res in results]) + [res.download() for res in results] + hashes = set([res.hash.hexdigest() for res in results]) + assert hashes == set(expected_hashes)