2021-02-07 17:46:20 +13:00
|
|
|
#!/usr/bin/env python3
|
2022-12-20 12:32:37 +13:00
|
|
|
# -*- coding: utf-8 -*-
|
2021-02-07 17:46:20 +13:00
|
|
|
|
2020-06-04 03:10:25 +12:00
|
|
|
import json
|
2021-03-21 14:10:06 +13:00
|
|
|
import re
|
2021-02-25 23:40:08 +13:00
|
|
|
from typing import Optional
|
2021-02-07 01:29:13 +13:00
|
|
|
|
2021-03-21 14:10:06 +13:00
|
|
|
import bs4
|
2021-02-11 12:10:40 +13:00
|
|
|
from praw.models import Submission
|
2020-05-29 06:42:11 +12:00
|
|
|
|
2021-04-23 23:06:16 +12:00
|
|
|
from bdfr.exceptions import SiteDownloaderError
|
2021-04-12 19:58:32 +12:00
|
|
|
from bdfr.resource import Resource
|
|
|
|
from bdfr.site_authenticator import SiteAuthenticator
|
|
|
|
from bdfr.site_downloaders.base_downloader import BaseDownloader
|
2020-05-29 06:42:11 +12:00
|
|
|
|
2021-02-25 22:32:06 +13:00
|
|
|
|
2021-02-07 14:33:19 +13:00
|
|
|
class Imgur(BaseDownloader):
|
2021-02-15 18:12:27 +13:00
|
|
|
def __init__(self, post: Submission):
|
|
|
|
super().__init__(post)
|
2021-02-07 17:46:20 +13:00
|
|
|
self.raw_data = {}
|
|
|
|
|
2021-02-26 21:57:05 +13:00
|
|
|
def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]:
|
2021-03-21 14:10:06 +13:00
|
|
|
self.raw_data = self._get_data(self.post.url)
|
2020-05-29 06:42:11 +12:00
|
|
|
|
2021-03-21 14:10:06 +13:00
|
|
|
out = []
|
2022-12-03 18:11:17 +13:00
|
|
|
if "album_images" in self.raw_data:
|
|
|
|
images = self.raw_data["album_images"]
|
|
|
|
for image in images["images"]:
|
2021-04-23 22:58:01 +12:00
|
|
|
out.append(self._compute_image_url(image))
|
2020-06-04 03:10:25 +12:00
|
|
|
else:
|
2021-04-23 22:58:01 +12:00
|
|
|
out.append(self._compute_image_url(self.raw_data))
|
2021-02-11 12:10:40 +13:00
|
|
|
return out
|
2020-06-02 00:05:02 +12:00
|
|
|
|
2021-04-23 22:58:01 +12:00
|
|
|
def _compute_image_url(self, image: dict) -> Resource:
|
2022-12-03 18:11:17 +13:00
|
|
|
ext = self._validate_extension(image["ext"])
|
|
|
|
if image.get("prefer_video", False):
|
|
|
|
ext = ".mp4"
|
2021-09-13 09:50:31 +12:00
|
|
|
|
2022-12-03 18:11:17 +13:00
|
|
|
image_url = "https://i.imgur.com/" + image["hash"] + ext
|
2021-07-27 16:02:30 +12:00
|
|
|
return Resource(self.post, image_url, Resource.retry_download(image_url))
|
2020-05-29 06:42:11 +12:00
|
|
|
|
2021-03-21 14:10:06 +13:00
|
|
|
@staticmethod
|
|
|
|
def _get_data(link: str) -> dict:
|
2022-09-19 15:27:17 +12:00
|
|
|
try:
|
2023-01-10 06:48:24 +13:00
|
|
|
if re.search(r".*/(.*?)(gallery/|a/)", link):
|
|
|
|
imgur_id = re.match(r".*/(?:gallery/|a/)(.*?)(?:/.*)?$", link).group(1)
|
|
|
|
else:
|
2023-01-10 09:34:14 +13:00
|
|
|
imgur_id = re.match(r".*/(.*?)(?:_d)?(?:\..{0,})?$", link).group(1)
|
|
|
|
gallery = "a/" if re.search(r".*/(.*?)(gallery/|a/)", link) else ""
|
|
|
|
if len(imgur_id) > 7:
|
|
|
|
if imgur_id.endswith(("s", "b", "t", "m", "l", "h")):
|
|
|
|
imgur_id = imgur_id[:7]
|
|
|
|
else:
|
|
|
|
raise SiteDownloaderError(f"Imgur ID error in link {link}")
|
|
|
|
link = f"https://imgur.com/{gallery}{imgur_id}"
|
2022-09-19 15:27:17 +12:00
|
|
|
except AttributeError:
|
2022-12-03 18:11:17 +13:00
|
|
|
raise SiteDownloaderError(f"Could not extract Imgur ID from {link}")
|
2021-03-28 13:15:21 +13:00
|
|
|
|
2022-12-03 18:11:17 +13:00
|
|
|
res = Imgur.retrieve_url(link, cookies={"over18": "1", "postpagebeta": "0"})
|
2020-06-04 03:10:25 +12:00
|
|
|
|
2022-12-03 18:11:17 +13:00
|
|
|
soup = bs4.BeautifulSoup(res.text, "html.parser")
|
|
|
|
scripts = soup.find_all("script", attrs={"type": "text/javascript"})
|
|
|
|
scripts = [script.string.replace("\n", "") for script in scripts if script.string]
|
2020-06-04 03:10:25 +12:00
|
|
|
|
2022-12-03 18:11:17 +13:00
|
|
|
script_regex = re.compile(r"\s*\(function\(widgetFactory\)\s*{\s*widgetFactory\.mergeConfig\(\'gallery\'")
|
2021-03-21 14:10:06 +13:00
|
|
|
chosen_script = list(filter(lambda s: re.search(script_regex, s), scripts))
|
|
|
|
if len(chosen_script) != 1:
|
2022-12-03 18:11:17 +13:00
|
|
|
raise SiteDownloaderError(f"Could not read page source from {link}")
|
2020-06-04 03:10:25 +12:00
|
|
|
|
2021-04-06 13:04:08 +12:00
|
|
|
chosen_script = chosen_script[0]
|
2021-03-21 14:10:06 +13:00
|
|
|
|
2022-12-03 18:11:17 +13:00
|
|
|
outer_regex = re.compile(r"widgetFactory\.mergeConfig\(\'gallery\', ({.*})\);")
|
|
|
|
inner_regex = re.compile(r"image\s*:(.*),\s*group")
|
2021-04-06 13:04:08 +12:00
|
|
|
try:
|
|
|
|
image_dict = re.search(outer_regex, chosen_script).group(1)
|
|
|
|
image_dict = re.search(inner_regex, image_dict).group(1)
|
|
|
|
except AttributeError:
|
2022-12-29 04:00:43 +13:00
|
|
|
raise SiteDownloaderError("Could not find image dictionary in page source")
|
2021-04-06 13:04:08 +12:00
|
|
|
|
|
|
|
try:
|
|
|
|
image_dict = json.loads(image_dict)
|
|
|
|
except json.JSONDecodeError as e:
|
2022-12-03 18:11:17 +13:00
|
|
|
raise SiteDownloaderError(f"Could not parse received dict as JSON: {e}")
|
2021-03-21 14:10:06 +13:00
|
|
|
|
|
|
|
return image_dict
|
2020-05-29 06:42:11 +12:00
|
|
|
|
|
|
|
@staticmethod
|
2021-02-07 17:46:20 +13:00
|
|
|
def _validate_extension(extension_suffix: str) -> str:
|
2022-12-03 18:11:17 +13:00
|
|
|
extension_suffix = re.sub(r"\?.*", "", extension_suffix)
|
|
|
|
possible_extensions = (".jpg", ".png", ".mp4", ".gif")
|
2021-03-21 14:10:06 +13:00
|
|
|
selection = [ext for ext in possible_extensions if ext == extension_suffix]
|
|
|
|
if len(selection) == 1:
|
|
|
|
return selection[0]
|
2021-02-06 21:35:50 +13:00
|
|
|
else:
|
2021-02-15 19:30:39 +13:00
|
|
|
raise SiteDownloaderError(f'"{extension_suffix}" is not recognized as a valid extension for Imgur')
|