From 5fbe64dc7128237329d6c56bc96ded954d9ccd28 Mon Sep 17 00:00:00 2001 From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com> Date: Sat, 21 Jan 2023 17:36:56 -0500 Subject: [PATCH] Move Imgur to API Moves Imgur to use API with public Client-ID. --- bdfr/site_downloaders/imgur.py | 76 +++++++-------------- tests/site_downloaders/test_imgur.py | 98 +--------------------------- 2 files changed, 24 insertions(+), 150 deletions(-) diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py index 294209d..bfcecc0 100644 --- a/bdfr/site_downloaders/imgur.py +++ b/bdfr/site_downloaders/imgur.py @@ -5,7 +5,6 @@ import json import re from typing import Optional -import bs4 from praw.models import Submission from bdfr.exceptions import SiteDownloaderError @@ -23,73 +22,42 @@ class Imgur(BaseDownloader): self.raw_data = self._get_data(self.post.url) out = [] - if "album_images" in self.raw_data: - images = self.raw_data["album_images"] - for image in images["images"]: - out.append(self._compute_image_url(image)) + if "is_album" in self.raw_data: + for image in self.raw_data["images"]: + if "mp4" in image: + out.append(Resource(self.post, image["mp4"], Resource.retry_download(image["mp4"]))) + else: + out.append(Resource(self.post, image["link"], Resource.retry_download(image["link"]))) else: - out.append(self._compute_image_url(self.raw_data)) + if "mp4" in self.raw_data: + out.append(Resource(self.post, self.raw_data["mp4"], Resource.retry_download(self.raw_data["mp4"]))) + else: + out.append(Resource(self.post, self.raw_data["link"], Resource.retry_download(self.raw_data["link"]))) return out - def _compute_image_url(self, image: dict) -> Resource: - ext = self._validate_extension(image["ext"]) - if image.get("prefer_video", False): - ext = ".mp4" - - image_url = "https://i.imgur.com/" + image["hash"] + ext - return Resource(self.post, image_url, Resource.retry_download(image_url)) - @staticmethod def _get_data(link: str) -> dict: try: if re.search(r".*/(.*?)(gallery/|a/)", link): imgur_id = re.match(r".*/(?:gallery/|a/)(.*?)(?:/.*)?$", link).group(1) + link = f"https://api.imgur.com/3/album/{imgur_id}" else: imgur_id = re.match(r".*/(.*?)(?:_d)?(?:\..{0,})?$", link).group(1) - gallery = "a/" if re.search(r".*/(.*?)(gallery/|a/)", link) else "" - if len(imgur_id) > 7: - if imgur_id.endswith(("s", "b", "t", "m", "l", "h")): - imgur_id = imgur_id[:7] - else: - raise SiteDownloaderError(f"Imgur ID error in link {link}") - link = f"https://imgur.com/{gallery}{imgur_id}" + link = f"https://api.imgur.com/3/image/{imgur_id}" except AttributeError: raise SiteDownloaderError(f"Could not extract Imgur ID from {link}") - res = Imgur.retrieve_url(link, cookies={"over18": "1", "postpagebeta": "0"}) - - soup = bs4.BeautifulSoup(res.text, "html.parser") - scripts = soup.find_all("script", attrs={"type": "text/javascript"}) - scripts = [script.string.replace("\n", "") for script in scripts if script.string] - - script_regex = re.compile(r"\s*\(function\(widgetFactory\)\s*{\s*widgetFactory\.mergeConfig\(\'gallery\'") - chosen_script = list(filter(lambda s: re.search(script_regex, s), scripts)) - if len(chosen_script) != 1: - raise SiteDownloaderError(f"Could not read page source from {link}") - - chosen_script = chosen_script[0] - - outer_regex = re.compile(r"widgetFactory\.mergeConfig\(\'gallery\', ({.*})\);") - inner_regex = re.compile(r"image\s*:(.*),\s*group") - try: - image_dict = re.search(outer_regex, chosen_script).group(1) - image_dict = re.search(inner_regex, image_dict).group(1) - except AttributeError: - raise SiteDownloaderError("Could not find image dictionary in page source") + headers = { + "referer": "https://imgur.com/", + "origin": "https://imgur.com", + "content-type": "application/json", + "Authorization": "Client-ID 546c25a59c58ad7", + } + res = Imgur.retrieve_url(link, headers=headers) try: - image_dict = json.loads(image_dict) + image_dict = json.loads(res.text) except json.JSONDecodeError as e: - raise SiteDownloaderError(f"Could not parse received dict as JSON: {e}") + raise SiteDownloaderError(f"Could not parse received response as JSON: {e}") - return image_dict - - @staticmethod - def _validate_extension(extension_suffix: str) -> str: - extension_suffix = re.sub(r"\?.*", "", extension_suffix) - possible_extensions = (".jpg", ".png", ".mp4", ".gif") - selection = [ext for ext in possible_extensions if ext == extension_suffix] - if len(selection) == 1: - return selection[0] - else: - raise SiteDownloaderError(f'"{extension_suffix}" is not recognized as a valid extension for Imgur') + return image_dict["data"] diff --git a/tests/site_downloaders/test_imgur.py b/tests/site_downloaders/test_imgur.py index 2e74a25..744488b 100644 --- a/tests/site_downloaders/test_imgur.py +++ b/tests/site_downloaders/test_imgur.py @@ -5,105 +5,10 @@ from unittest.mock import Mock import pytest -from bdfr.exceptions import SiteDownloaderError from bdfr.resource import Resource from bdfr.site_downloaders.imgur import Imgur -@pytest.mark.online -@pytest.mark.parametrize( - ("test_url", "expected_gen_dict", "expected_image_dict"), - ( - ( - "https://imgur.com/a/xWZsDDP", - {"num_images": "1", "id": "xWZsDDP", "hash": "xWZsDDP"}, - [{"hash": "ypa8YfS", "title": "", "ext": ".png", "animated": False}], - ), - ( - "https://imgur.com/gallery/IjJJdlC", - {"num_images": 1, "id": 384898055, "hash": "IjJJdlC"}, - [ - { - "hash": "CbbScDt", - "description": "watch when he gets it", - "ext": ".gif", - "animated": True, - "has_sound": False, - } - ], - ), - ( - "https://imgur.com/a/dcc84Gt", - {"num_images": "4", "id": "dcc84Gt", "hash": "dcc84Gt"}, - [ - {"hash": "ylx0Kle", "ext": ".jpg", "title": ""}, - {"hash": "TdYfKbK", "ext": ".jpg", "title": ""}, - {"hash": "pCxGbe8", "ext": ".jpg", "title": ""}, - {"hash": "TSAkikk", "ext": ".jpg", "title": ""}, - ], - ), - ( - "https://m.imgur.com/a/py3RW0j", - { - "num_images": "1", - "id": "py3RW0j", - "hash": "py3RW0j", - }, - [{"hash": "K24eQmK", "has_sound": False, "ext": ".jpg"}], - ), - ), -) -def test_get_data_album(test_url: str, expected_gen_dict: dict, expected_image_dict: list[dict]): - result = Imgur._get_data(test_url) - assert all([result.get(key) == expected_gen_dict[key] for key in expected_gen_dict.keys()]) - - # Check if all the keys from the test dict are correct in at least one of the album entries - assert any( - [ - all([image.get(key) == image_dict[key] for key in image_dict.keys()]) - for image_dict in expected_image_dict - for image in result["album_images"]["images"] - ] - ) - - -@pytest.mark.online -@pytest.mark.parametrize( - ("test_url", "expected_image_dict"), - ( - ("https://i.imgur.com/dLk3FGY.gifv", {"hash": "dLk3FGY", "title": "", "ext": ".mp4", "animated": True}), - ( - "https://imgur.com/65FqTpT.gifv", - {"hash": "65FqTpT", "title": "", "description": "", "animated": True, "mimetype": "video/mp4"}, - ), - ), -) -def test_get_data_gif(test_url: str, expected_image_dict: dict): - result = Imgur._get_data(test_url) - assert all([result.get(key) == expected_image_dict[key] for key in expected_image_dict.keys()]) - - -@pytest.mark.parametrize("test_extension", (".gif", ".png", ".jpg", ".mp4")) -def test_imgur_extension_validation_good(test_extension: str): - result = Imgur._validate_extension(test_extension) - assert result == test_extension - - -@pytest.mark.parametrize( - "test_extension", - ( - ".jpeg", - "bad", - ".avi", - ".test", - ".flac", - ), -) -def test_imgur_extension_validation_bad(test_extension: str): - with pytest.raises(SiteDownloaderError): - Imgur._validate_extension(test_extension) - - @pytest.mark.online @pytest.mark.parametrize( ("test_url", "expected_hashes"), @@ -130,7 +35,7 @@ def test_imgur_extension_validation_bad(test_extension: str): ("https://i.imgur.com/lFJai6i.gifv", ("01a6e79a30bec0e644e5da12365d5071",)), ("https://i.imgur.com/ywSyILa.gifv?", ("56d4afc32d2966017c38d98568709b45",)), ("https://imgur.com/ubYwpbk.GIFV", ("d4a774aac1667783f9ed3a1bd02fac0c",)), - ("https://i.imgur.com/j1CNCZY.gifv", ("58e7e6d972058c18b7ecde910ca147e3",)), + ("https://i.imgur.com/j1CNCZY.gifv", ("ed63d7062bc32edaeea8b53f876a307c",)), ("https://i.imgur.com/uTvtQsw.gifv", ("46c86533aa60fc0e09f2a758513e3ac2",)), ("https://i.imgur.com/OGeVuAe.giff", ("77389679084d381336f168538793f218",)), ("https://i.imgur.com/OGeVuAe.gift", ("77389679084d381336f168538793f218",)), @@ -142,6 +47,7 @@ def test_imgur_extension_validation_bad(test_extension: str): ("https://imgur.com/a/1qzfWtY/gifv", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)), ("https://imgur.com/a/1qzfWtY/mp4", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)), ("https://imgur.com/a/1qzfWtY/spqr", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)), + ("https://i.imgur.com/expO7Rc.gifv", ("e309f98158fc98072eb2ae68f947f421",)), ), ) def test_find_resources(test_url: str, expected_hashes: list[str]):