2021-02-07 17:46:20 +13:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
import logging
|
|
|
|
import re
|
2021-02-06 21:35:50 +13:00
|
|
|
import urllib.error
|
2020-05-29 06:42:11 +12:00
|
|
|
import urllib.request
|
|
|
|
from html.parser import HTMLParser
|
2021-02-25 23:40:08 +13:00
|
|
|
from typing import Optional
|
2020-05-29 06:42:11 +12:00
|
|
|
|
2021-02-11 12:10:40 +13:00
|
|
|
from praw.models import Submission
|
|
|
|
|
2021-02-26 21:57:05 +13:00
|
|
|
from bulkredditdownloader.site_authenticator import SiteAuthenticator
|
2021-03-05 16:32:24 +13:00
|
|
|
from bulkredditdownloader.exceptions import NotADownloadableLinkError
|
2021-02-25 23:40:08 +13:00
|
|
|
from bulkredditdownloader.resource import Resource
|
2021-02-07 20:08:24 +13:00
|
|
|
from bulkredditdownloader.site_downloaders.base_downloader import BaseDownloader
|
2021-02-07 17:46:20 +13:00
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
2020-05-29 06:42:11 +12:00
|
|
|
|
2021-02-25 22:32:06 +13:00
|
|
|
|
2021-02-07 14:33:19 +13:00
|
|
|
class Erome(BaseDownloader):
|
2021-02-15 18:12:27 +13:00
|
|
|
def __init__(self, post: Submission):
|
|
|
|
super().__init__(post)
|
2021-02-07 17:46:20 +13:00
|
|
|
|
2021-02-26 21:57:05 +13:00
|
|
|
def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]:
|
2020-05-29 06:42:11 +12:00
|
|
|
try:
|
2021-02-27 13:35:43 +13:00
|
|
|
images = set(self._get_links(self.post.url))
|
2020-05-29 06:42:11 +12:00
|
|
|
except urllib.error.HTTPError:
|
|
|
|
raise NotADownloadableLinkError("Not a downloadable link")
|
|
|
|
|
2021-02-11 12:10:40 +13:00
|
|
|
if len(images) == 1:
|
2020-05-29 06:42:11 +12:00
|
|
|
|
2021-02-27 13:35:43 +13:00
|
|
|
image = images.pop()
|
2021-02-07 17:46:20 +13:00
|
|
|
if not re.match(r'https?://.*', image):
|
|
|
|
image = "https://" + image
|
2021-02-25 23:40:08 +13:00
|
|
|
return [Resource(self.post, image)]
|
2020-05-29 06:42:11 +12:00
|
|
|
|
|
|
|
else:
|
2021-02-11 12:10:40 +13:00
|
|
|
out = []
|
2021-02-07 17:46:20 +13:00
|
|
|
for i, image in enumerate(images):
|
|
|
|
if not re.match(r'https?://.*', image):
|
|
|
|
image = "https://" + image
|
2021-02-27 13:35:43 +13:00
|
|
|
out.append(Resource(self.post, image))
|
2021-02-11 12:10:40 +13:00
|
|
|
return out
|
2021-02-25 22:32:06 +13:00
|
|
|
|
2021-02-07 17:46:20 +13:00
|
|
|
@staticmethod
|
|
|
|
def _get_links(url: str) -> list[str]:
|
2020-05-29 06:42:11 +12:00
|
|
|
content = []
|
2021-02-06 21:35:50 +13:00
|
|
|
line_number = None
|
2020-05-29 06:42:11 +12:00
|
|
|
|
2021-02-07 17:46:20 +13:00
|
|
|
# TODO: move to bs4 and requests
|
2020-05-29 06:42:11 +12:00
|
|
|
class EromeParser(HTMLParser):
|
|
|
|
tag = None
|
2021-02-25 22:32:06 +13:00
|
|
|
|
2020-05-29 06:42:11 +12:00
|
|
|
def handle_starttag(self, tag, attrs):
|
2021-02-25 22:32:06 +13:00
|
|
|
self.tag = {tag: {attr[0]: attr[1] for attr in attrs}}
|
2020-05-29 06:42:11 +12:00
|
|
|
|
2021-02-06 21:35:50 +13:00
|
|
|
page_source = (urllib.request.urlopen(url).read().decode().split('\n'))
|
2020-05-29 06:42:11 +12:00
|
|
|
|
|
|
|
""" FIND WHERE ALBUM STARTS IN ORDER NOT TO GET WRONG LINKS"""
|
2021-02-06 21:35:50 +13:00
|
|
|
for i in range(len(page_source)):
|
2020-05-29 06:42:11 +12:00
|
|
|
obj = EromeParser()
|
2021-02-06 21:35:50 +13:00
|
|
|
obj.feed(page_source[i])
|
2020-05-29 06:42:11 +12:00
|
|
|
tag = obj.tag
|
2021-02-25 22:32:06 +13:00
|
|
|
|
2020-05-29 06:42:11 +12:00
|
|
|
if tag is not None:
|
|
|
|
if "div" in tag:
|
|
|
|
if "id" in tag["div"]:
|
|
|
|
if tag["div"]["id"] == "album":
|
2021-02-06 21:35:50 +13:00
|
|
|
line_number = i
|
2020-05-29 06:42:11 +12:00
|
|
|
break
|
|
|
|
|
2021-02-06 21:35:50 +13:00
|
|
|
for line in page_source[line_number:]:
|
2020-05-29 06:42:11 +12:00
|
|
|
obj = EromeParser()
|
|
|
|
obj.feed(line)
|
|
|
|
tag = obj.tag
|
|
|
|
if tag is not None:
|
|
|
|
if "img" in tag:
|
|
|
|
if "class" in tag["img"]:
|
2021-02-25 22:32:06 +13:00
|
|
|
if tag["img"]["class"] == "img-front":
|
2020-05-29 06:42:11 +12:00
|
|
|
content.append(tag["img"]["src"])
|
|
|
|
elif "source" in tag:
|
|
|
|
content.append(tag["source"]["src"])
|
2021-02-25 22:32:06 +13:00
|
|
|
|
2021-02-06 21:35:50 +13:00
|
|
|
return [link for link in content if link.endswith("_480p.mp4") or not link.endswith(".mp4")]
|