v1.7.0 (#97)
* tools file name change to utils * Seperate downloaders (#94) * Seperated the downloaders * Remove redundant code * Changed file names * refactor * Redgifs (#95) * Init commit * Init commit * GifDeliveryNetwork (#96) * Initial commit * Gfycat forwarding to GDN bug fixed
This commit is contained in:
parent
c5a2eed80f
commit
148e20d1d6
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -5,3 +5,4 @@ __pycache__/
|
||||||
src/__pycache__/
|
src/__pycache__/
|
||||||
config.json
|
config.json
|
||||||
env/
|
env/
|
||||||
|
.vscode/
|
||||||
|
|
17
script.py
17
script.py
|
@ -14,11 +14,17 @@ import webbrowser
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
from pathlib import Path, PurePath
|
from pathlib import Path, PurePath
|
||||||
|
|
||||||
from src.downloader import Direct, Erome, Gfycat, Imgur, Self
|
from src.downloaders.Direct import Direct
|
||||||
|
from src.downloaders.Erome import Erome
|
||||||
|
from src.downloaders.Gfycat import Gfycat
|
||||||
|
from src.downloaders.Imgur import Imgur
|
||||||
|
from src.downloaders.redgifs import Redgifs
|
||||||
|
from src.downloaders.selfPost import SelfPost
|
||||||
|
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
|
||||||
from src.errors import *
|
from src.errors import *
|
||||||
from src.parser import LinkDesigner
|
from src.parser import LinkDesigner
|
||||||
from src.searcher import getPosts
|
from src.searcher import getPosts
|
||||||
from src.tools import (GLOBAL, createLogFile, jsonFile, nameCorrector,
|
from src.utils import (GLOBAL, createLogFile, jsonFile, nameCorrector,
|
||||||
printToFile)
|
printToFile)
|
||||||
|
|
||||||
__author__ = "Ali Parlakci"
|
__author__ = "Ali Parlakci"
|
||||||
|
@ -496,7 +502,8 @@ def downloadPost(SUBMISSION):
|
||||||
global lastRequestTime
|
global lastRequestTime
|
||||||
|
|
||||||
downloaders = {
|
downloaders = {
|
||||||
"imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":Self
|
"imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":SelfPost,
|
||||||
|
"redgifs":Redgifs, "gifdeliverynetwork": GifDeliveryNetwork
|
||||||
}
|
}
|
||||||
|
|
||||||
print()
|
print()
|
||||||
|
@ -532,7 +539,7 @@ def downloadPost(SUBMISSION):
|
||||||
if not (credit['UserRemaining'] == 0 or \
|
if not (credit['UserRemaining'] == 0 or \
|
||||||
credit['ClientRemaining'] == 0):
|
credit['ClientRemaining'] == 0):
|
||||||
|
|
||||||
"""This block of code is needed
|
"""This block of code is needed for API workaround
|
||||||
"""
|
"""
|
||||||
while int(time.time() - lastRequestTime) <= 2:
|
while int(time.time() - lastRequestTime) <= 2:
|
||||||
pass
|
pass
|
||||||
|
@ -568,7 +575,7 @@ def download(submissions):
|
||||||
FAILED_FILE = createLogFile("FAILED")
|
FAILED_FILE = createLogFile("FAILED")
|
||||||
|
|
||||||
for i in range(subsLenght):
|
for i in range(subsLenght):
|
||||||
print(f"\n({i+1}/{subsLenght}) – r/{submissions[i]['postSubreddit']}",
|
print(f"\n({i+1}/{subsLenght}) – {submissions[i]['postId']} – r/{submissions[i]['postSubreddit']}",
|
||||||
end="")
|
end="")
|
||||||
print(f" – {submissions[i]['postType'].upper()}",end="",noPrint=True)
|
print(f" – {submissions[i]['postType'].upper()}",end="",noPrint=True)
|
||||||
|
|
||||||
|
|
|
@ -1,536 +0,0 @@
|
||||||
import io
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import urllib.request
|
|
||||||
from html.parser import HTMLParser
|
|
||||||
from multiprocessing import Queue
|
|
||||||
from pathlib import Path
|
|
||||||
from urllib.error import HTTPError
|
|
||||||
|
|
||||||
import imgurpython
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
|
|
||||||
from src.errors import (AlbumNotDownloadedCompletely, FileAlreadyExistsError,
|
|
||||||
FileNameTooLong, ImgurLoginError,
|
|
||||||
NotADownloadableLinkError)
|
|
||||||
from src.tools import GLOBAL, nameCorrector, printToFile
|
|
||||||
|
|
||||||
VanillaPrint = print
|
|
||||||
print = printToFile
|
|
||||||
|
|
||||||
def dlProgress(count, blockSize, totalSize):
|
|
||||||
"""Function for writing download progress to console
|
|
||||||
"""
|
|
||||||
|
|
||||||
downloadedMbs = int(count*blockSize*(10**(-6)))
|
|
||||||
fileSize = int(totalSize*(10**(-6)))
|
|
||||||
sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs,fileSize))
|
|
||||||
sys.stdout.flush()
|
|
||||||
|
|
||||||
def getExtension(link):
|
|
||||||
"""Extract file extension from image link.
|
|
||||||
If didn't find any, return '.jpg'
|
|
||||||
"""
|
|
||||||
|
|
||||||
imageTypes = ['jpg','png','mp4','webm','gif']
|
|
||||||
parsed = link.split('.')
|
|
||||||
for TYPE in imageTypes:
|
|
||||||
if TYPE in parsed:
|
|
||||||
return "."+parsed[-1]
|
|
||||||
else:
|
|
||||||
if not "v.redd.it" in link:
|
|
||||||
return '.jpg'
|
|
||||||
else:
|
|
||||||
return '.mp4'
|
|
||||||
|
|
||||||
def getFile(fileDir,tempDir,imageURL,indent=0):
|
|
||||||
"""Downloads given file to given directory.
|
|
||||||
|
|
||||||
fileDir -- Full file directory
|
|
||||||
tempDir -- Full file directory with the extension of '.tmp'
|
|
||||||
imageURL -- URL to the file to be downloaded
|
|
||||||
|
|
||||||
redditID -- Post's reddit id if renaming the file is necessary.
|
|
||||||
As too long file names seem not working.
|
|
||||||
"""
|
|
||||||
|
|
||||||
headers = [
|
|
||||||
("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
|
|
||||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 "\
|
|
||||||
"Safari/537.36 OPR/54.0.2952.64"),
|
|
||||||
("Accept", "text/html,application/xhtml+xml,application/xml;" \
|
|
||||||
"q=0.9,image/webp,image/apng,*/*;q=0.8"),
|
|
||||||
("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.3"),
|
|
||||||
("Accept-Encoding", "none"),
|
|
||||||
("Accept-Language", "en-US,en;q=0.8"),
|
|
||||||
("Connection", "keep-alive")
|
|
||||||
]
|
|
||||||
|
|
||||||
opener = urllib.request.build_opener()
|
|
||||||
if not "imgur" in imageURL:
|
|
||||||
opener.addheaders = headers
|
|
||||||
urllib.request.install_opener(opener)
|
|
||||||
|
|
||||||
if not (os.path.isfile(fileDir)):
|
|
||||||
for i in range(3):
|
|
||||||
try:
|
|
||||||
urllib.request.urlretrieve(imageURL,
|
|
||||||
tempDir,
|
|
||||||
reporthook=dlProgress)
|
|
||||||
os.rename(tempDir,fileDir)
|
|
||||||
except ConnectionResetError as exception:
|
|
||||||
print(" "*indent + str(exception))
|
|
||||||
print(" "*indent + "Trying again\n")
|
|
||||||
except FileNotFoundError:
|
|
||||||
raise FileNameTooLong
|
|
||||||
else:
|
|
||||||
print(" "*indent+"Downloaded"+" "*10)
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
raise FileAlreadyExistsError
|
|
||||||
|
|
||||||
class Erome:
|
|
||||||
def __init__(self,directory,post):
|
|
||||||
try:
|
|
||||||
IMAGES = self.getLinks(post['postURL'])
|
|
||||||
except urllib.error.HTTPError:
|
|
||||||
raise NotADownloadableLinkError("Not a downloadable link")
|
|
||||||
|
|
||||||
imagesLenght = len(IMAGES)
|
|
||||||
howManyDownloaded = imagesLenght
|
|
||||||
duplicates = 0
|
|
||||||
|
|
||||||
if imagesLenght == 1:
|
|
||||||
|
|
||||||
extension = getExtension(IMAGES[0])
|
|
||||||
|
|
||||||
"""Filenames are declared here"""
|
|
||||||
|
|
||||||
title = nameCorrector(post['postTitle'])
|
|
||||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+extension)
|
|
||||||
|
|
||||||
fileDir = directory / (
|
|
||||||
post["postSubmitter"]+"_"+title+"_"+post['postId']+extension
|
|
||||||
)
|
|
||||||
tempDir = directory / (
|
|
||||||
post["postSubmitter"]+"_"+title+"_"+post['postId']+".tmp"
|
|
||||||
)
|
|
||||||
|
|
||||||
imageURL = IMAGES[0]
|
|
||||||
if 'https://' not in imageURL and 'http://' not in imageURL:
|
|
||||||
imageURL = "https://" + imageURL
|
|
||||||
|
|
||||||
try:
|
|
||||||
getFile(fileDir,tempDir,imageURL)
|
|
||||||
except FileNameTooLong:
|
|
||||||
fileDir = directory / (post['postId'] + extension)
|
|
||||||
tempDir = directory / (post['postId'] + '.tmp')
|
|
||||||
getFile(fileDir,tempDir,imageURL)
|
|
||||||
|
|
||||||
else:
|
|
||||||
title = nameCorrector(post['postTitle'])
|
|
||||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
|
||||||
|
|
||||||
folderDir = directory / (
|
|
||||||
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
if not os.path.exists(folderDir):
|
|
||||||
os.makedirs(folderDir)
|
|
||||||
except FileNotFoundError:
|
|
||||||
folderDir = directory / post['postId']
|
|
||||||
os.makedirs(folderDir)
|
|
||||||
|
|
||||||
for i in range(imagesLenght):
|
|
||||||
|
|
||||||
extension = getExtension(IMAGES[i])
|
|
||||||
|
|
||||||
fileName = str(i+1)
|
|
||||||
imageURL = IMAGES[i]
|
|
||||||
if 'https://' not in imageURL and 'http://' not in imageURL:
|
|
||||||
imageURL = "https://" + imageURL
|
|
||||||
|
|
||||||
fileDir = folderDir / (fileName + extension)
|
|
||||||
tempDir = folderDir / (fileName + ".tmp")
|
|
||||||
|
|
||||||
print(" ({}/{})".format(i+1,imagesLenght))
|
|
||||||
print(" {}".format(fileName+extension))
|
|
||||||
|
|
||||||
try:
|
|
||||||
getFile(fileDir,tempDir,imageURL,indent=2)
|
|
||||||
print()
|
|
||||||
except FileAlreadyExistsError:
|
|
||||||
print(" The file already exists" + " "*10,end="\n\n")
|
|
||||||
duplicates += 1
|
|
||||||
howManyDownloaded -= 1
|
|
||||||
|
|
||||||
except Exception as exception:
|
|
||||||
# raise exception
|
|
||||||
print("\n Could not get the file")
|
|
||||||
print(
|
|
||||||
" "
|
|
||||||
+ "{class_name}: {info}".format(
|
|
||||||
class_name=exception.__class__.__name__,
|
|
||||||
info=str(exception)
|
|
||||||
)
|
|
||||||
+ "\n"
|
|
||||||
)
|
|
||||||
exceptionType = exception
|
|
||||||
howManyDownloaded -= 1
|
|
||||||
|
|
||||||
if duplicates == imagesLenght:
|
|
||||||
raise FileAlreadyExistsError
|
|
||||||
elif howManyDownloaded + duplicates < imagesLenght:
|
|
||||||
raise AlbumNotDownloadedCompletely(
|
|
||||||
"Album Not Downloaded Completely"
|
|
||||||
)
|
|
||||||
|
|
||||||
def getLinks(self,url,lineNumber=129):
|
|
||||||
|
|
||||||
content = []
|
|
||||||
lineNumber = None
|
|
||||||
|
|
||||||
class EromeParser(HTMLParser):
|
|
||||||
tag = None
|
|
||||||
def handle_starttag(self, tag, attrs):
|
|
||||||
self.tag = {tag:{attr[0]: attr[1] for attr in attrs}}
|
|
||||||
|
|
||||||
pageSource = (urllib.request.urlopen(url).read().decode().split('\n'))
|
|
||||||
|
|
||||||
""" FIND WHERE ALBUM STARTS IN ORDER NOT TO GET WRONG LINKS"""
|
|
||||||
for i in range(len(pageSource)):
|
|
||||||
obj = EromeParser()
|
|
||||||
obj.feed(pageSource[i])
|
|
||||||
tag = obj.tag
|
|
||||||
|
|
||||||
if tag is not None:
|
|
||||||
if "div" in tag:
|
|
||||||
if "id" in tag["div"]:
|
|
||||||
if tag["div"]["id"] == "album":
|
|
||||||
lineNumber = i
|
|
||||||
break
|
|
||||||
|
|
||||||
for line in pageSource[lineNumber:]:
|
|
||||||
obj = EromeParser()
|
|
||||||
obj.feed(line)
|
|
||||||
tag = obj.tag
|
|
||||||
if tag is not None:
|
|
||||||
if "img" in tag:
|
|
||||||
if "class" in tag["img"]:
|
|
||||||
if tag["img"]["class"]=="img-front":
|
|
||||||
content.append(tag["img"]["src"])
|
|
||||||
elif "source" in tag:
|
|
||||||
content.append(tag["source"]["src"])
|
|
||||||
|
|
||||||
return [
|
|
||||||
link for link in content \
|
|
||||||
if link.endswith("_480p.mp4") or not link.endswith(".mp4")
|
|
||||||
]
|
|
||||||
|
|
||||||
class Imgur:
|
|
||||||
def __init__(self,directory,post):
|
|
||||||
self.imgurClient = self.initImgur()
|
|
||||||
|
|
||||||
imgurID = self.getId(post['postURL'])
|
|
||||||
content = self.getLink(imgurID)
|
|
||||||
|
|
||||||
if not os.path.exists(directory): os.makedirs(directory)
|
|
||||||
|
|
||||||
if content['type'] == 'image':
|
|
||||||
|
|
||||||
try:
|
|
||||||
post['mediaURL'] = content['object'].mp4
|
|
||||||
except AttributeError:
|
|
||||||
post['mediaURL'] = content['object'].link
|
|
||||||
|
|
||||||
post['postExt'] = getExtension(post['mediaURL'])
|
|
||||||
|
|
||||||
title = nameCorrector(post['postTitle'])
|
|
||||||
|
|
||||||
"""Filenames are declared here"""
|
|
||||||
|
|
||||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+post['postExt'])
|
|
||||||
|
|
||||||
fileDir = directory / (
|
|
||||||
post["postSubmitter"]
|
|
||||||
+ "_" + title
|
|
||||||
+ "_" + post['postId']
|
|
||||||
+ post['postExt']
|
|
||||||
)
|
|
||||||
|
|
||||||
tempDir = directory / (
|
|
||||||
post["postSubmitter"]
|
|
||||||
+ "_" + title
|
|
||||||
+ "_" + post['postId']
|
|
||||||
+ ".tmp"
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
getFile(fileDir,tempDir,post['mediaURL'])
|
|
||||||
except FileNameTooLong:
|
|
||||||
fileDir = directory / post['postId'] + post['postExt']
|
|
||||||
tempDir = directory / post['postId'] + '.tmp'
|
|
||||||
getFile(fileDir,tempDir,post['mediaURL'])
|
|
||||||
|
|
||||||
elif content['type'] == 'album':
|
|
||||||
exceptionType = ""
|
|
||||||
images = content['object'].images
|
|
||||||
imagesLenght = len(images)
|
|
||||||
howManyDownloaded = imagesLenght
|
|
||||||
duplicates = 0
|
|
||||||
|
|
||||||
title = nameCorrector(post['postTitle'])
|
|
||||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
|
||||||
|
|
||||||
folderDir = directory / (
|
|
||||||
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
if not os.path.exists(folderDir):
|
|
||||||
os.makedirs(folderDir)
|
|
||||||
except FileNotFoundError:
|
|
||||||
folderDir = directory / post['postId']
|
|
||||||
os.makedirs(folderDir)
|
|
||||||
|
|
||||||
for i in range(imagesLenght):
|
|
||||||
try:
|
|
||||||
imageURL = images[i]['mp4']
|
|
||||||
except KeyError:
|
|
||||||
imageURL = images[i]['link']
|
|
||||||
|
|
||||||
images[i]['Ext'] = getExtension(imageURL)
|
|
||||||
|
|
||||||
fileName = (str(i+1)
|
|
||||||
+ "_"
|
|
||||||
+ nameCorrector(str(images[i]['title']))
|
|
||||||
+ "_"
|
|
||||||
+ images[i]['id'])
|
|
||||||
|
|
||||||
"""Filenames are declared here"""
|
|
||||||
|
|
||||||
fileDir = folderDir / (fileName + images[i]['Ext'])
|
|
||||||
tempDir = folderDir / (fileName + ".tmp")
|
|
||||||
|
|
||||||
print(" ({}/{})".format(i+1,imagesLenght))
|
|
||||||
print(" {}".format(fileName+images[i]['Ext']))
|
|
||||||
|
|
||||||
try:
|
|
||||||
getFile(fileDir,tempDir,imageURL,indent=2)
|
|
||||||
print()
|
|
||||||
except FileAlreadyExistsError:
|
|
||||||
print(" The file already exists" + " "*10,end="\n\n")
|
|
||||||
duplicates += 1
|
|
||||||
howManyDownloaded -= 1
|
|
||||||
|
|
||||||
# IF FILE NAME IS TOO LONG, IT WONT REGISTER
|
|
||||||
except FileNameTooLong:
|
|
||||||
fileName = (str(i+1) + "_" + images[i]['id'])
|
|
||||||
fileDir = folderDir / (fileName + images[i]['Ext'])
|
|
||||||
tempDir = folderDir / (fileName + ".tmp")
|
|
||||||
try:
|
|
||||||
getFile(fileDir,tempDir,imageURL,indent=2)
|
|
||||||
# IF STILL TOO LONG
|
|
||||||
except FileNameTooLong:
|
|
||||||
fileName = str(i+1)
|
|
||||||
fileDir = folderDir / (fileName + images[i]['Ext'])
|
|
||||||
tempDir = folderDir / (fileName + ".tmp")
|
|
||||||
getFile(fileDir,tempDir,imageURL,indent=2)
|
|
||||||
|
|
||||||
except Exception as exception:
|
|
||||||
print("\n Could not get the file")
|
|
||||||
print(
|
|
||||||
" "
|
|
||||||
+ "{class_name}: {info}".format(
|
|
||||||
class_name=exception.__class__.__name__,
|
|
||||||
info=str(exception)
|
|
||||||
)
|
|
||||||
+ "\n"
|
|
||||||
)
|
|
||||||
exceptionType = exception
|
|
||||||
howManyDownloaded -= 1
|
|
||||||
|
|
||||||
if duplicates == imagesLenght:
|
|
||||||
raise FileAlreadyExistsError
|
|
||||||
elif howManyDownloaded + duplicates < imagesLenght:
|
|
||||||
raise AlbumNotDownloadedCompletely(
|
|
||||||
"Album Not Downloaded Completely"
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def initImgur():
|
|
||||||
"""Initialize imgur api"""
|
|
||||||
|
|
||||||
config = GLOBAL.config
|
|
||||||
return imgurpython.ImgurClient(
|
|
||||||
config['imgur_client_id'],
|
|
||||||
config['imgur_client_secret']
|
|
||||||
)
|
|
||||||
def getId(self,submissionURL):
|
|
||||||
"""Extract imgur post id
|
|
||||||
and determine if its a single image or album
|
|
||||||
"""
|
|
||||||
|
|
||||||
domainLenght = len("imgur.com/")
|
|
||||||
if submissionURL[-1] == "/":
|
|
||||||
submissionURL = submissionURL[:-1]
|
|
||||||
|
|
||||||
if "a/" in submissionURL or "gallery/" in submissionURL:
|
|
||||||
albumId = submissionURL.split("/")[-1]
|
|
||||||
return {'id':albumId, 'type':'album'}
|
|
||||||
|
|
||||||
else:
|
|
||||||
url = submissionURL.replace('.','/').split('/')
|
|
||||||
imageId = url[url.index('com')+1]
|
|
||||||
return {'id':imageId, 'type':'image'}
|
|
||||||
|
|
||||||
def getLink(self,identity):
|
|
||||||
"""Request imgur object from imgur api
|
|
||||||
"""
|
|
||||||
|
|
||||||
if identity['type'] == 'image':
|
|
||||||
return {'object':self.imgurClient.get_image(identity['id']),
|
|
||||||
'type':'image'}
|
|
||||||
elif identity['type'] == 'album':
|
|
||||||
return {'object':self.imgurClient.get_album(identity['id']),
|
|
||||||
'type':'album'}
|
|
||||||
@staticmethod
|
|
||||||
def get_credits():
|
|
||||||
return Imgur.initImgur().get_credits()
|
|
||||||
|
|
||||||
class Gfycat:
|
|
||||||
def __init__(self,directory,POST):
|
|
||||||
try:
|
|
||||||
POST['mediaURL'] = self.getLink(POST['postURL'])
|
|
||||||
except IndexError:
|
|
||||||
raise NotADownloadableLinkError("Could not read the page source")
|
|
||||||
except Exception as exception:
|
|
||||||
#debug
|
|
||||||
raise exception
|
|
||||||
raise NotADownloadableLinkError("Could not read the page source")
|
|
||||||
|
|
||||||
POST['postExt'] = getExtension(POST['mediaURL'])
|
|
||||||
|
|
||||||
if not os.path.exists(directory): os.makedirs(directory)
|
|
||||||
title = nameCorrector(POST['postTitle'])
|
|
||||||
|
|
||||||
"""Filenames are declared here"""
|
|
||||||
|
|
||||||
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
|
||||||
|
|
||||||
fileDir = directory / (
|
|
||||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
|
||||||
)
|
|
||||||
tempDir = directory / (
|
|
||||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
getFile(fileDir,tempDir,POST['mediaURL'])
|
|
||||||
except FileNameTooLong:
|
|
||||||
fileDir = directory / (POST['postId']+POST['postExt'])
|
|
||||||
tempDir = directory / (POST['postId']+".tmp")
|
|
||||||
|
|
||||||
getFile(fileDir,tempDir,POST['mediaURL'])
|
|
||||||
|
|
||||||
def getLink(self, url, query='<source id="mp4Source" src=', lineNumber=105):
|
|
||||||
"""Extract direct link to the video from page's source
|
|
||||||
and return it
|
|
||||||
"""
|
|
||||||
|
|
||||||
if '.webm' in url or '.mp4' in url or '.gif' in url:
|
|
||||||
return url
|
|
||||||
|
|
||||||
if url[-1:] == '/':
|
|
||||||
url = url[:-1]
|
|
||||||
|
|
||||||
url = "https://gfycat.com/" + url.split('/')[-1]
|
|
||||||
|
|
||||||
pageSource = (urllib.request.urlopen(url).read().decode())
|
|
||||||
|
|
||||||
soup = BeautifulSoup(pageSource, "html.parser")
|
|
||||||
attributes = {"data-react-helmet":"true","type":"application/ld+json"}
|
|
||||||
content = soup.find("script",attrs=attributes)
|
|
||||||
|
|
||||||
if content is None:
|
|
||||||
raise NotADownloadableLinkError("Could not read the page source")
|
|
||||||
|
|
||||||
return json.loads(content.text)["video"]["contentUrl"]
|
|
||||||
|
|
||||||
class Direct:
|
|
||||||
def __init__(self,directory,POST):
|
|
||||||
POST['postExt'] = getExtension(POST['postURL'])
|
|
||||||
if not os.path.exists(directory): os.makedirs(directory)
|
|
||||||
title = nameCorrector(POST['postTitle'])
|
|
||||||
|
|
||||||
"""Filenames are declared here"""
|
|
||||||
|
|
||||||
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
|
||||||
|
|
||||||
fileDir = directory / (
|
|
||||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
|
||||||
)
|
|
||||||
tempDir = directory / (
|
|
||||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
getFile(fileDir,tempDir,POST['postURL'])
|
|
||||||
except FileNameTooLong:
|
|
||||||
fileDir = directory / (POST['postId']+POST['postExt'])
|
|
||||||
tempDir = directory / (POST['postId']+".tmp")
|
|
||||||
|
|
||||||
getFile(fileDir,tempDir,POST['postURL'])
|
|
||||||
|
|
||||||
class Self:
|
|
||||||
def __init__(self,directory,post):
|
|
||||||
if not os.path.exists(directory): os.makedirs(directory)
|
|
||||||
|
|
||||||
title = nameCorrector(post['postTitle'])
|
|
||||||
|
|
||||||
"""Filenames are declared here"""
|
|
||||||
|
|
||||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+".md")
|
|
||||||
|
|
||||||
fileDir = directory / (
|
|
||||||
post["postSubmitter"]+"_"+title+"_"+post['postId']+".md"
|
|
||||||
)
|
|
||||||
|
|
||||||
if Path.is_file(fileDir):
|
|
||||||
raise FileAlreadyExistsError
|
|
||||||
|
|
||||||
try:
|
|
||||||
self.writeToFile(fileDir,post)
|
|
||||||
except FileNotFoundError:
|
|
||||||
fileDir = post['postId']+".md"
|
|
||||||
fileDir = directory / fileDir
|
|
||||||
|
|
||||||
self.writeToFile(fileDir,post)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def writeToFile(directory,post):
|
|
||||||
|
|
||||||
"""Self posts are formatted here"""
|
|
||||||
content = ("## ["
|
|
||||||
+ post["postTitle"]
|
|
||||||
+ "]("
|
|
||||||
+ post["postURL"]
|
|
||||||
+ ")\n"
|
|
||||||
+ post["postContent"]
|
|
||||||
+ "\n\n---\n\n"
|
|
||||||
+ "submitted to [r/"
|
|
||||||
+ post["postSubreddit"]
|
|
||||||
+ "](https://www.reddit.com/r/"
|
|
||||||
+ post["postSubreddit"]
|
|
||||||
+ ") by [u/"
|
|
||||||
+ post["postSubmitter"]
|
|
||||||
+ "](https://www.reddit.com/user/"
|
|
||||||
+ post["postSubmitter"]
|
|
||||||
+ ")")
|
|
||||||
|
|
||||||
with io.open(directory,"w",encoding="utf-8") as FILE:
|
|
||||||
VanillaPrint(content,file=FILE)
|
|
||||||
|
|
||||||
print("Downloaded")
|
|
32
src/downloaders/Direct.py
Normal file
32
src/downloaders/Direct.py
Normal file
|
@ -0,0 +1,32 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
from src.downloaders.downloaderUtils import getFile, getExtension
|
||||||
|
|
||||||
|
from src.errors import FileNameTooLong
|
||||||
|
from src.utils import nameCorrector
|
||||||
|
from src.utils import printToFile as print
|
||||||
|
|
||||||
|
class Direct:
|
||||||
|
def __init__(self,directory,POST):
|
||||||
|
POST['postExt'] = getExtension(POST['postURL'])
|
||||||
|
if not os.path.exists(directory): os.makedirs(directory)
|
||||||
|
title = nameCorrector(POST['postTitle'])
|
||||||
|
|
||||||
|
"""Filenames are declared here"""
|
||||||
|
|
||||||
|
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||||
|
|
||||||
|
fileDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||||
|
)
|
||||||
|
tempDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
getFile(fileDir,tempDir,POST['postURL'])
|
||||||
|
except FileNameTooLong:
|
||||||
|
fileDir = directory / (POST['postId']+POST['postExt'])
|
||||||
|
tempDir = directory / (POST['postId']+".tmp")
|
||||||
|
|
||||||
|
getFile(fileDir,tempDir,POST['postURL'])
|
149
src/downloaders/Erome.py
Normal file
149
src/downloaders/Erome.py
Normal file
|
@ -0,0 +1,149 @@
|
||||||
|
import os
|
||||||
|
import urllib.request
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
|
||||||
|
from src.downloaders.downloaderUtils import getFile
|
||||||
|
from src.downloaders.downloaderUtils import getExtension
|
||||||
|
|
||||||
|
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely,
|
||||||
|
NotADownloadableLinkError, FileAlreadyExistsError)
|
||||||
|
from src.utils import nameCorrector
|
||||||
|
from src.utils import printToFile as print
|
||||||
|
|
||||||
|
class Erome:
|
||||||
|
def __init__(self,directory,post):
|
||||||
|
try:
|
||||||
|
IMAGES = self.getLinks(post['postURL'])
|
||||||
|
except urllib.error.HTTPError:
|
||||||
|
raise NotADownloadableLinkError("Not a downloadable link")
|
||||||
|
|
||||||
|
imagesLenght = len(IMAGES)
|
||||||
|
howManyDownloaded = imagesLenght
|
||||||
|
duplicates = 0
|
||||||
|
|
||||||
|
if imagesLenght == 1:
|
||||||
|
|
||||||
|
extension = getExtension(IMAGES[0])
|
||||||
|
|
||||||
|
"""Filenames are declared here"""
|
||||||
|
|
||||||
|
title = nameCorrector(post['postTitle'])
|
||||||
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+extension)
|
||||||
|
|
||||||
|
fileDir = directory / (
|
||||||
|
post["postSubmitter"]+"_"+title+"_"+post['postId']+extension
|
||||||
|
)
|
||||||
|
tempDir = directory / (
|
||||||
|
post["postSubmitter"]+"_"+title+"_"+post['postId']+".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
|
imageURL = IMAGES[0]
|
||||||
|
if 'https://' not in imageURL and 'http://' not in imageURL:
|
||||||
|
imageURL = "https://" + imageURL
|
||||||
|
|
||||||
|
try:
|
||||||
|
getFile(fileDir,tempDir,imageURL)
|
||||||
|
except FileNameTooLong:
|
||||||
|
fileDir = directory / (post['postId'] + extension)
|
||||||
|
tempDir = directory / (post['postId'] + '.tmp')
|
||||||
|
getFile(fileDir,tempDir,imageURL)
|
||||||
|
|
||||||
|
else:
|
||||||
|
title = nameCorrector(post['postTitle'])
|
||||||
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
||||||
|
|
||||||
|
folderDir = directory / (
|
||||||
|
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if not os.path.exists(folderDir):
|
||||||
|
os.makedirs(folderDir)
|
||||||
|
except FileNotFoundError:
|
||||||
|
folderDir = directory / post['postId']
|
||||||
|
os.makedirs(folderDir)
|
||||||
|
|
||||||
|
for i in range(imagesLenght):
|
||||||
|
|
||||||
|
extension = getExtension(IMAGES[i])
|
||||||
|
|
||||||
|
fileName = str(i+1)
|
||||||
|
imageURL = IMAGES[i]
|
||||||
|
if 'https://' not in imageURL and 'http://' not in imageURL:
|
||||||
|
imageURL = "https://" + imageURL
|
||||||
|
|
||||||
|
fileDir = folderDir / (fileName + extension)
|
||||||
|
tempDir = folderDir / (fileName + ".tmp")
|
||||||
|
|
||||||
|
print(" ({}/{})".format(i+1,imagesLenght))
|
||||||
|
print(" {}".format(fileName+extension))
|
||||||
|
|
||||||
|
try:
|
||||||
|
getFile(fileDir,tempDir,imageURL,indent=2)
|
||||||
|
print()
|
||||||
|
except FileAlreadyExistsError:
|
||||||
|
print(" The file already exists" + " "*10,end="\n\n")
|
||||||
|
duplicates += 1
|
||||||
|
howManyDownloaded -= 1
|
||||||
|
|
||||||
|
except Exception as exception:
|
||||||
|
# raise exception
|
||||||
|
print("\n Could not get the file")
|
||||||
|
print(
|
||||||
|
" "
|
||||||
|
+ "{class_name}: {info}".format(
|
||||||
|
class_name=exception.__class__.__name__,
|
||||||
|
info=str(exception)
|
||||||
|
)
|
||||||
|
+ "\n"
|
||||||
|
)
|
||||||
|
howManyDownloaded -= 1
|
||||||
|
|
||||||
|
if duplicates == imagesLenght:
|
||||||
|
raise FileAlreadyExistsError
|
||||||
|
elif howManyDownloaded + duplicates < imagesLenght:
|
||||||
|
raise AlbumNotDownloadedCompletely(
|
||||||
|
"Album Not Downloaded Completely"
|
||||||
|
)
|
||||||
|
|
||||||
|
def getLinks(self,url,lineNumber=129):
|
||||||
|
|
||||||
|
content = []
|
||||||
|
lineNumber = None
|
||||||
|
|
||||||
|
class EromeParser(HTMLParser):
|
||||||
|
tag = None
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
self.tag = {tag:{attr[0]: attr[1] for attr in attrs}}
|
||||||
|
|
||||||
|
pageSource = (urllib.request.urlopen(url).read().decode().split('\n'))
|
||||||
|
|
||||||
|
""" FIND WHERE ALBUM STARTS IN ORDER NOT TO GET WRONG LINKS"""
|
||||||
|
for i in range(len(pageSource)):
|
||||||
|
obj = EromeParser()
|
||||||
|
obj.feed(pageSource[i])
|
||||||
|
tag = obj.tag
|
||||||
|
|
||||||
|
if tag is not None:
|
||||||
|
if "div" in tag:
|
||||||
|
if "id" in tag["div"]:
|
||||||
|
if tag["div"]["id"] == "album":
|
||||||
|
lineNumber = i
|
||||||
|
break
|
||||||
|
|
||||||
|
for line in pageSource[lineNumber:]:
|
||||||
|
obj = EromeParser()
|
||||||
|
obj.feed(line)
|
||||||
|
tag = obj.tag
|
||||||
|
if tag is not None:
|
||||||
|
if "img" in tag:
|
||||||
|
if "class" in tag["img"]:
|
||||||
|
if tag["img"]["class"]=="img-front":
|
||||||
|
content.append(tag["img"]["src"])
|
||||||
|
elif "source" in tag:
|
||||||
|
content.append(tag["source"]["src"])
|
||||||
|
|
||||||
|
return [
|
||||||
|
link for link in content \
|
||||||
|
if link.endswith("_480p.mp4") or not link.endswith(".mp4")
|
||||||
|
]
|
67
src/downloaders/Gfycat.py
Normal file
67
src/downloaders/Gfycat.py
Normal file
|
@ -0,0 +1,67 @@
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import urllib.request
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from src.downloaders.downloaderUtils import getFile, getExtension
|
||||||
|
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely,
|
||||||
|
NotADownloadableLinkError, FileAlreadyExistsError)
|
||||||
|
from src.utils import nameCorrector
|
||||||
|
from src.utils import printToFile as print
|
||||||
|
from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork
|
||||||
|
|
||||||
|
class Gfycat:
|
||||||
|
def __init__(self,directory,POST):
|
||||||
|
try:
|
||||||
|
POST['mediaURL'] = self.getLink(POST['postURL'])
|
||||||
|
except IndexError:
|
||||||
|
raise NotADownloadableLinkError("Could not read the page source")
|
||||||
|
|
||||||
|
POST['postExt'] = getExtension(POST['mediaURL'])
|
||||||
|
|
||||||
|
if not os.path.exists(directory): os.makedirs(directory)
|
||||||
|
title = nameCorrector(POST['postTitle'])
|
||||||
|
|
||||||
|
"""Filenames are declared here"""
|
||||||
|
|
||||||
|
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||||
|
|
||||||
|
fileDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||||
|
)
|
||||||
|
tempDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
getFile(fileDir,tempDir,POST['mediaURL'])
|
||||||
|
except FileNameTooLong:
|
||||||
|
fileDir = directory / (POST['postId']+POST['postExt'])
|
||||||
|
tempDir = directory / (POST['postId']+".tmp")
|
||||||
|
|
||||||
|
getFile(fileDir,tempDir,POST['mediaURL'])
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def getLink(url):
|
||||||
|
"""Extract direct link to the video from page's source
|
||||||
|
and return it
|
||||||
|
"""
|
||||||
|
|
||||||
|
if '.webm' in url or '.mp4' in url or '.gif' in url:
|
||||||
|
return url
|
||||||
|
|
||||||
|
if url[-1:] == '/':
|
||||||
|
url = url[:-1]
|
||||||
|
|
||||||
|
url = "https://gfycat.com/" + url.split('/')[-1]
|
||||||
|
|
||||||
|
pageSource = (urllib.request.urlopen(url).read().decode())
|
||||||
|
|
||||||
|
soup = BeautifulSoup(pageSource, "html.parser")
|
||||||
|
attributes = {"data-react-helmet":"true","type":"application/ld+json"}
|
||||||
|
content = soup.find("script",attrs=attributes)
|
||||||
|
|
||||||
|
if content is None:
|
||||||
|
return GifDeliveryNetwork.getLink(url)
|
||||||
|
|
||||||
|
return json.loads(content.contents[0])["video"]["contentUrl"]
|
178
src/downloaders/Imgur.py
Normal file
178
src/downloaders/Imgur.py
Normal file
|
@ -0,0 +1,178 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
import imgurpython
|
||||||
|
|
||||||
|
from src.downloaders.downloaderUtils import getExtension, getFile
|
||||||
|
from src.errors import (AlbumNotDownloadedCompletely, FileAlreadyExistsError,
|
||||||
|
FileNameTooLong)
|
||||||
|
from src.utils import GLOBAL, nameCorrector
|
||||||
|
from src.utils import printToFile as print
|
||||||
|
|
||||||
|
|
||||||
|
class Imgur:
|
||||||
|
def __init__(self,directory,post):
|
||||||
|
self.imgurClient = self.initImgur()
|
||||||
|
|
||||||
|
imgurID = self.getId(post['postURL'])
|
||||||
|
content = self.getLink(imgurID)
|
||||||
|
|
||||||
|
if not os.path.exists(directory): os.makedirs(directory)
|
||||||
|
|
||||||
|
if content['type'] == 'image':
|
||||||
|
|
||||||
|
try:
|
||||||
|
post['mediaURL'] = content['object'].mp4
|
||||||
|
except AttributeError:
|
||||||
|
post['mediaURL'] = content['object'].link
|
||||||
|
|
||||||
|
post['postExt'] = getExtension(post['mediaURL'])
|
||||||
|
|
||||||
|
title = nameCorrector(post['postTitle'])
|
||||||
|
|
||||||
|
"""Filenames are declared here"""
|
||||||
|
|
||||||
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+post['postExt'])
|
||||||
|
|
||||||
|
fileDir = directory / (
|
||||||
|
post["postSubmitter"]
|
||||||
|
+ "_" + title
|
||||||
|
+ "_" + post['postId']
|
||||||
|
+ post['postExt']
|
||||||
|
)
|
||||||
|
|
||||||
|
tempDir = directory / (
|
||||||
|
post["postSubmitter"]
|
||||||
|
+ "_" + title
|
||||||
|
+ "_" + post['postId']
|
||||||
|
+ ".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
getFile(fileDir,tempDir,post['mediaURL'])
|
||||||
|
except FileNameTooLong:
|
||||||
|
fileDir = directory / post['postId'] + post['postExt']
|
||||||
|
tempDir = directory / post['postId'] + '.tmp'
|
||||||
|
getFile(fileDir,tempDir,post['mediaURL'])
|
||||||
|
|
||||||
|
elif content['type'] == 'album':
|
||||||
|
images = content['object'].images
|
||||||
|
imagesLenght = len(images)
|
||||||
|
howManyDownloaded = imagesLenght
|
||||||
|
duplicates = 0
|
||||||
|
|
||||||
|
title = nameCorrector(post['postTitle'])
|
||||||
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
||||||
|
|
||||||
|
folderDir = directory / (
|
||||||
|
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if not os.path.exists(folderDir):
|
||||||
|
os.makedirs(folderDir)
|
||||||
|
except FileNotFoundError:
|
||||||
|
folderDir = directory / post['postId']
|
||||||
|
os.makedirs(folderDir)
|
||||||
|
|
||||||
|
for i in range(imagesLenght):
|
||||||
|
try:
|
||||||
|
imageURL = images[i]['mp4']
|
||||||
|
except KeyError:
|
||||||
|
imageURL = images[i]['link']
|
||||||
|
|
||||||
|
images[i]['Ext'] = getExtension(imageURL)
|
||||||
|
|
||||||
|
fileName = (str(i+1)
|
||||||
|
+ "_"
|
||||||
|
+ nameCorrector(str(images[i]['title']))
|
||||||
|
+ "_"
|
||||||
|
+ images[i]['id'])
|
||||||
|
|
||||||
|
"""Filenames are declared here"""
|
||||||
|
|
||||||
|
fileDir = folderDir / (fileName + images[i]['Ext'])
|
||||||
|
tempDir = folderDir / (fileName + ".tmp")
|
||||||
|
|
||||||
|
print(" ({}/{})".format(i+1,imagesLenght))
|
||||||
|
print(" {}".format(fileName+images[i]['Ext']))
|
||||||
|
|
||||||
|
try:
|
||||||
|
getFile(fileDir,tempDir,imageURL,indent=2)
|
||||||
|
print()
|
||||||
|
except FileAlreadyExistsError:
|
||||||
|
print(" The file already exists" + " "*10,end="\n\n")
|
||||||
|
duplicates += 1
|
||||||
|
howManyDownloaded -= 1
|
||||||
|
|
||||||
|
# IF FILE NAME IS TOO LONG, IT WONT REGISTER
|
||||||
|
except FileNameTooLong:
|
||||||
|
fileName = (str(i+1) + "_" + images[i]['id'])
|
||||||
|
fileDir = folderDir / (fileName + images[i]['Ext'])
|
||||||
|
tempDir = folderDir / (fileName + ".tmp")
|
||||||
|
try:
|
||||||
|
getFile(fileDir,tempDir,imageURL,indent=2)
|
||||||
|
# IF STILL TOO LONG
|
||||||
|
except FileNameTooLong:
|
||||||
|
fileName = str(i+1)
|
||||||
|
fileDir = folderDir / (fileName + images[i]['Ext'])
|
||||||
|
tempDir = folderDir / (fileName + ".tmp")
|
||||||
|
getFile(fileDir,tempDir,imageURL,indent=2)
|
||||||
|
|
||||||
|
except Exception as exception:
|
||||||
|
print("\n Could not get the file")
|
||||||
|
print(
|
||||||
|
" "
|
||||||
|
+ "{class_name}: {info}".format(
|
||||||
|
class_name=exception.__class__.__name__,
|
||||||
|
info=str(exception)
|
||||||
|
)
|
||||||
|
+ "\n"
|
||||||
|
)
|
||||||
|
howManyDownloaded -= 1
|
||||||
|
|
||||||
|
if duplicates == imagesLenght:
|
||||||
|
raise FileAlreadyExistsError
|
||||||
|
elif howManyDownloaded + duplicates < imagesLenght:
|
||||||
|
raise AlbumNotDownloadedCompletely(
|
||||||
|
"Album Not Downloaded Completely"
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def initImgur():
|
||||||
|
"""Initialize imgur api"""
|
||||||
|
|
||||||
|
config = GLOBAL.config
|
||||||
|
return imgurpython.ImgurClient(
|
||||||
|
config['imgur_client_id'],
|
||||||
|
config['imgur_client_secret']
|
||||||
|
)
|
||||||
|
def getId(self,submissionURL):
|
||||||
|
"""Extract imgur post id
|
||||||
|
and determine if its a single image or album
|
||||||
|
"""
|
||||||
|
|
||||||
|
if submissionURL[-1] == "/":
|
||||||
|
submissionURL = submissionURL[:-1]
|
||||||
|
|
||||||
|
if "a/" in submissionURL or "gallery/" in submissionURL:
|
||||||
|
albumId = submissionURL.split("/")[-1]
|
||||||
|
return {'id':albumId, 'type':'album'}
|
||||||
|
|
||||||
|
else:
|
||||||
|
url = submissionURL.replace('.','/').split('/')
|
||||||
|
imageId = url[url.index('com')+1]
|
||||||
|
return {'id':imageId, 'type':'image'}
|
||||||
|
|
||||||
|
def getLink(self,identity):
|
||||||
|
"""Request imgur object from imgur api
|
||||||
|
"""
|
||||||
|
|
||||||
|
if identity['type'] == 'image':
|
||||||
|
return {'object':self.imgurClient.get_image(identity['id']),
|
||||||
|
'type':'image'}
|
||||||
|
elif identity['type'] == 'album':
|
||||||
|
return {'object':self.imgurClient.get_album(identity['id']),
|
||||||
|
'type':'album'}
|
||||||
|
@staticmethod
|
||||||
|
def get_credits():
|
||||||
|
return Imgur.initImgur().get_credits()
|
0
src/downloaders/__init__.py
Normal file
0
src/downloaders/__init__.py
Normal file
77
src/downloaders/downloaderUtils.py
Normal file
77
src/downloaders/downloaderUtils.py
Normal file
|
@ -0,0 +1,77 @@
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from src.errors import FileAlreadyExistsError, FileNameTooLong
|
||||||
|
|
||||||
|
def dlProgress(count, blockSize, totalSize):
|
||||||
|
"""Function for writing download progress to console
|
||||||
|
"""
|
||||||
|
|
||||||
|
downloadedMbs = int(count*blockSize*(10**(-6)))
|
||||||
|
fileSize = int(totalSize*(10**(-6)))
|
||||||
|
sys.stdout.write("{}Mb/{}Mb\r".format(downloadedMbs,fileSize))
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
def getExtension(link):
|
||||||
|
"""Extract file extension from image link.
|
||||||
|
If didn't find any, return '.jpg'
|
||||||
|
"""
|
||||||
|
|
||||||
|
imageTypes = ['jpg','png','mp4','webm','gif']
|
||||||
|
parsed = link.split('.')
|
||||||
|
for fileType in imageTypes:
|
||||||
|
if fileType in parsed:
|
||||||
|
return "."+parsed[-1]
|
||||||
|
else:
|
||||||
|
if not "v.redd.it" in link:
|
||||||
|
return '.jpg'
|
||||||
|
else:
|
||||||
|
return '.mp4'
|
||||||
|
|
||||||
|
def getFile(fileDir,tempDir,imageURL,indent=0):
|
||||||
|
"""Downloads given file to given directory.
|
||||||
|
|
||||||
|
fileDir -- Full file directory
|
||||||
|
tempDir -- Full file directory with the extension of '.tmp'
|
||||||
|
imageURL -- URL to the file to be downloaded
|
||||||
|
|
||||||
|
redditID -- Post's reddit id if renaming the file is necessary.
|
||||||
|
As too long file names seem not working.
|
||||||
|
"""
|
||||||
|
|
||||||
|
headers = [
|
||||||
|
("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 "\
|
||||||
|
"Safari/537.36 OPR/54.0.2952.64"),
|
||||||
|
("Accept", "text/html,application/xhtml+xml,application/xml;" \
|
||||||
|
"q=0.9,image/webp,image/apng,*/*;q=0.8"),
|
||||||
|
("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.3"),
|
||||||
|
("Accept-Encoding", "none"),
|
||||||
|
("Accept-Language", "en-US,en;q=0.8"),
|
||||||
|
("Connection", "keep-alive")
|
||||||
|
]
|
||||||
|
|
||||||
|
opener = urllib.request.build_opener()
|
||||||
|
if not "imgur" in imageURL:
|
||||||
|
opener.addheaders = headers
|
||||||
|
urllib.request.install_opener(opener)
|
||||||
|
|
||||||
|
if not (os.path.isfile(fileDir)):
|
||||||
|
for i in range(3):
|
||||||
|
try:
|
||||||
|
urllib.request.urlretrieve(imageURL,
|
||||||
|
tempDir,
|
||||||
|
reporthook=dlProgress)
|
||||||
|
os.rename(tempDir,fileDir)
|
||||||
|
except ConnectionResetError as exception:
|
||||||
|
print(" "*indent + str(exception))
|
||||||
|
print(" "*indent + "Trying again\n")
|
||||||
|
except FileNotFoundError:
|
||||||
|
raise FileNameTooLong
|
||||||
|
else:
|
||||||
|
print(" "*indent+"Downloaded"+" "*10)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
raise FileAlreadyExistsError
|
67
src/downloaders/gifDeliveryNetwork.py
Normal file
67
src/downloaders/gifDeliveryNetwork.py
Normal file
|
@ -0,0 +1,67 @@
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import urllib.request
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from src.downloaders.downloaderUtils import getFile, getExtension
|
||||||
|
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely,
|
||||||
|
NotADownloadableLinkError, FileAlreadyExistsError)
|
||||||
|
from src.utils import nameCorrector
|
||||||
|
from src.utils import printToFile as print
|
||||||
|
|
||||||
|
class GifDeliveryNetwork:
|
||||||
|
def __init__(self,directory,POST):
|
||||||
|
try:
|
||||||
|
POST['mediaURL'] = self.getLink(POST['postURL'])
|
||||||
|
except IndexError:
|
||||||
|
raise NotADownloadableLinkError("Could not read the page source")
|
||||||
|
|
||||||
|
POST['postExt'] = getExtension(POST['mediaURL'])
|
||||||
|
|
||||||
|
if not os.path.exists(directory): os.makedirs(directory)
|
||||||
|
title = nameCorrector(POST['postTitle'])
|
||||||
|
|
||||||
|
"""Filenames are declared here"""
|
||||||
|
|
||||||
|
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||||
|
|
||||||
|
fileDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||||
|
)
|
||||||
|
tempDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
getFile(fileDir,tempDir,POST['mediaURL'])
|
||||||
|
except FileNameTooLong:
|
||||||
|
fileDir = directory / (POST['postId']+POST['postExt'])
|
||||||
|
tempDir = directory / (POST['postId']+".tmp")
|
||||||
|
|
||||||
|
getFile(fileDir,tempDir,POST['mediaURL'])
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def getLink(url):
|
||||||
|
"""Extract direct link to the video from page's source
|
||||||
|
and return it
|
||||||
|
"""
|
||||||
|
|
||||||
|
if '.webm' in url.split('/')[-1] or '.mp4' in url.split('/')[-1] or '.gif' in url.split('/')[-1]:
|
||||||
|
return url
|
||||||
|
|
||||||
|
if url[-1:] == '/':
|
||||||
|
url = url[:-1]
|
||||||
|
|
||||||
|
url = "https://www.gifdeliverynetwork.com/" + url.split('/')[-1]
|
||||||
|
|
||||||
|
pageSource = (urllib.request.urlopen(url).read().decode())
|
||||||
|
|
||||||
|
soup = BeautifulSoup(pageSource, "html.parser")
|
||||||
|
attributes = {"id":"mp4Source","type":"video/mp4"}
|
||||||
|
content = soup.find("source",attrs=attributes)
|
||||||
|
|
||||||
|
if content is None:
|
||||||
|
|
||||||
|
raise NotADownloadableLinkError("Could not read the page source")
|
||||||
|
|
||||||
|
return content["src"]
|
65
src/downloaders/redgifs.py
Normal file
65
src/downloaders/redgifs.py
Normal file
|
@ -0,0 +1,65 @@
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import urllib.request
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from src.downloaders.downloaderUtils import getFile, getExtension
|
||||||
|
from src.errors import (FileNameTooLong, AlbumNotDownloadedCompletely,
|
||||||
|
NotADownloadableLinkError, FileAlreadyExistsError)
|
||||||
|
from src.utils import nameCorrector
|
||||||
|
from src.utils import printToFile as print
|
||||||
|
|
||||||
|
class Redgifs:
|
||||||
|
def __init__(self,directory,POST):
|
||||||
|
try:
|
||||||
|
POST['mediaURL'] = self.getLink(POST['postURL'])
|
||||||
|
except IndexError:
|
||||||
|
raise NotADownloadableLinkError("Could not read the page source")
|
||||||
|
|
||||||
|
POST['postExt'] = getExtension(POST['mediaURL'])
|
||||||
|
|
||||||
|
if not os.path.exists(directory): os.makedirs(directory)
|
||||||
|
title = nameCorrector(POST['postTitle'])
|
||||||
|
|
||||||
|
"""Filenames are declared here"""
|
||||||
|
|
||||||
|
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||||
|
|
||||||
|
fileDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||||
|
)
|
||||||
|
tempDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
getFile(fileDir,tempDir,POST['mediaURL'])
|
||||||
|
except FileNameTooLong:
|
||||||
|
fileDir = directory / (POST['postId']+POST['postExt'])
|
||||||
|
tempDir = directory / (POST['postId']+".tmp")
|
||||||
|
|
||||||
|
getFile(fileDir,tempDir,POST['mediaURL'])
|
||||||
|
|
||||||
|
def getLink(self, url):
|
||||||
|
"""Extract direct link to the video from page's source
|
||||||
|
and return it
|
||||||
|
"""
|
||||||
|
|
||||||
|
if '.webm' in url or '.mp4' in url or '.gif' in url:
|
||||||
|
return url
|
||||||
|
|
||||||
|
if url[-1:] == '/':
|
||||||
|
url = url[:-1]
|
||||||
|
|
||||||
|
url = "https://redgifs.com/watch/" + url.split('/')[-1]
|
||||||
|
|
||||||
|
pageSource = (urllib.request.urlopen(url).read().decode())
|
||||||
|
|
||||||
|
soup = BeautifulSoup(pageSource, "html.parser")
|
||||||
|
attributes = {"data-react-helmet":"true","type":"application/ld+json"}
|
||||||
|
content = soup.find("script",attrs=attributes)
|
||||||
|
|
||||||
|
if content is None:
|
||||||
|
raise NotADownloadableLinkError("Could not read the page source")
|
||||||
|
|
||||||
|
return json.loads(content.contents[0])["video"]["contentUrl"]
|
60
src/downloaders/selfPost.py
Normal file
60
src/downloaders/selfPost.py
Normal file
|
@ -0,0 +1,60 @@
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from src.errors import FileAlreadyExistsError
|
||||||
|
from src.utils import nameCorrector
|
||||||
|
|
||||||
|
VanillaPrint = print
|
||||||
|
from src.utils import printToFile as print
|
||||||
|
|
||||||
|
class SelfPost:
|
||||||
|
def __init__(self,directory,post):
|
||||||
|
if not os.path.exists(directory): os.makedirs(directory)
|
||||||
|
|
||||||
|
title = nameCorrector(post['postTitle'])
|
||||||
|
|
||||||
|
"""Filenames are declared here"""
|
||||||
|
|
||||||
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+".md")
|
||||||
|
|
||||||
|
fileDir = directory / (
|
||||||
|
post["postSubmitter"]+"_"+title+"_"+post['postId']+".md"
|
||||||
|
)
|
||||||
|
|
||||||
|
if Path.is_file(fileDir):
|
||||||
|
raise FileAlreadyExistsError
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.writeToFile(fileDir,post)
|
||||||
|
except FileNotFoundError:
|
||||||
|
fileDir = post['postId']+".md"
|
||||||
|
fileDir = directory / fileDir
|
||||||
|
|
||||||
|
self.writeToFile(fileDir,post)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def writeToFile(directory,post):
|
||||||
|
|
||||||
|
"""Self posts are formatted here"""
|
||||||
|
content = ("## ["
|
||||||
|
+ post["postTitle"]
|
||||||
|
+ "]("
|
||||||
|
+ post["postURL"]
|
||||||
|
+ ")\n"
|
||||||
|
+ post["postContent"]
|
||||||
|
+ "\n\n---\n\n"
|
||||||
|
+ "submitted to [r/"
|
||||||
|
+ post["postSubreddit"]
|
||||||
|
+ "](https://www.reddit.com/r/"
|
||||||
|
+ post["postSubreddit"]
|
||||||
|
+ ") by [u/"
|
||||||
|
+ post["postSubmitter"]
|
||||||
|
+ "](https://www.reddit.com/user/"
|
||||||
|
+ post["postSubmitter"]
|
||||||
|
+ ")")
|
||||||
|
|
||||||
|
with io.open(directory,"w",encoding="utf-8") as FILE:
|
||||||
|
VanillaPrint(content,file=FILE)
|
||||||
|
|
||||||
|
print("Downloaded")
|
|
@ -9,7 +9,7 @@ from urllib.error import HTTPError
|
||||||
import praw
|
import praw
|
||||||
from prawcore.exceptions import NotFound, ResponseException, Forbidden
|
from prawcore.exceptions import NotFound, ResponseException, Forbidden
|
||||||
|
|
||||||
from src.tools import GLOBAL, createLogFile, jsonFile, printToFile
|
from src.utils import GLOBAL, createLogFile, jsonFile, printToFile
|
||||||
from src.errors import (NoMatchingSubmissionFound, NoPrawSupport,
|
from src.errors import (NoMatchingSubmissionFound, NoPrawSupport,
|
||||||
NoRedditSupport, MultiredditNotFound,
|
NoRedditSupport, MultiredditNotFound,
|
||||||
InvalidSortingType, RedditLoginFailed,
|
InvalidSortingType, RedditLoginFailed,
|
||||||
|
@ -35,12 +35,12 @@ def beginPraw(config,user_agent = str(socket.gethostname())):
|
||||||
server.close()
|
server.close()
|
||||||
return client
|
return client
|
||||||
|
|
||||||
def send_message(self, message):
|
def send_message(self, client, message):
|
||||||
"""Send message to client and close the connection."""
|
"""Send message to client and close the connection."""
|
||||||
self.client.send(
|
client.send(
|
||||||
'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8')
|
'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8')
|
||||||
)
|
)
|
||||||
self.client.close()
|
client.close()
|
||||||
|
|
||||||
def getRefreshToken(self,*scopes):
|
def getRefreshToken(self,*scopes):
|
||||||
state = str(random.randint(0, 65000))
|
state = str(random.randint(0, 65000))
|
||||||
|
@ -48,8 +48,8 @@ def beginPraw(config,user_agent = str(socket.gethostname())):
|
||||||
print("Go to this URL and login to reddit:\n\n",url)
|
print("Go to this URL and login to reddit:\n\n",url)
|
||||||
webbrowser.open(url,new=2)
|
webbrowser.open(url,new=2)
|
||||||
|
|
||||||
self.client = self.recieve_connection()
|
client = self.recieve_connection()
|
||||||
data = self.client.recv(1024).decode('utf-8')
|
data = client.recv(1024).decode('utf-8')
|
||||||
str(data)
|
str(data)
|
||||||
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
|
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
|
||||||
params = {
|
params = {
|
||||||
|
@ -67,7 +67,7 @@ def beginPraw(config,user_agent = str(socket.gethostname())):
|
||||||
raise RedditLoginFailed
|
raise RedditLoginFailed
|
||||||
|
|
||||||
refresh_token = self.redditInstance.auth.authorize(params['code'])
|
refresh_token = self.redditInstance.auth.authorize(params['code'])
|
||||||
self.send_message(
|
self.send_message(client,
|
||||||
"<script>" \
|
"<script>" \
|
||||||
"alert(\"You can go back to terminal window now.\");" \
|
"alert(\"You can go back to terminal window now.\");" \
|
||||||
"</script>"
|
"</script>"
|
||||||
|
@ -299,10 +299,14 @@ def redditSearcher(posts,SINGLE_POST=False):
|
||||||
orderCount = 0
|
orderCount = 0
|
||||||
global gfycatCount
|
global gfycatCount
|
||||||
gfycatCount = 0
|
gfycatCount = 0
|
||||||
|
global redgifsCount
|
||||||
|
redgifsCount = 0
|
||||||
global imgurCount
|
global imgurCount
|
||||||
imgurCount = 0
|
imgurCount = 0
|
||||||
global eromeCount
|
global eromeCount
|
||||||
eromeCount = 0
|
eromeCount = 0
|
||||||
|
global gifDeliveryNetworkCount
|
||||||
|
gifDeliveryNetworkCount = 0
|
||||||
global directCount
|
global directCount
|
||||||
directCount = 0
|
directCount = 0
|
||||||
global selfCount
|
global selfCount
|
||||||
|
@ -394,9 +398,11 @@ def redditSearcher(posts,SINGLE_POST=False):
|
||||||
|
|
||||||
def checkIfMatching(submission):
|
def checkIfMatching(submission):
|
||||||
global gfycatCount
|
global gfycatCount
|
||||||
|
global redgifsCount
|
||||||
global imgurCount
|
global imgurCount
|
||||||
global eromeCount
|
global eromeCount
|
||||||
global directCount
|
global directCount
|
||||||
|
global gifDeliveryNetworkCount
|
||||||
global selfCount
|
global selfCount
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -424,6 +430,16 @@ def checkIfMatching(submission):
|
||||||
eromeCount += 1
|
eromeCount += 1
|
||||||
return details
|
return details
|
||||||
|
|
||||||
|
elif 'redgifs' in submission.domain:
|
||||||
|
details['postType'] = 'redgifs'
|
||||||
|
redgifsCount += 1
|
||||||
|
return details
|
||||||
|
|
||||||
|
elif 'gifdeliverynetwork' in submission.domain:
|
||||||
|
details['postType'] = 'gifdeliverynetwork'
|
||||||
|
gifDeliveryNetworkCount += 1
|
||||||
|
return details
|
||||||
|
|
||||||
elif submission.is_self:
|
elif submission.is_self:
|
||||||
details['postType'] = 'self'
|
details['postType'] = 'self'
|
||||||
details['postContent'] = submission.selftext
|
details['postContent'] = submission.selftext
|
||||||
|
@ -495,7 +511,7 @@ def isDirectLink(URL):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
for extension in imageTypes:
|
for extension in imageTypes:
|
||||||
if extension in URL:
|
if extension in URL.split("/")[-1]:
|
||||||
return URL
|
return URL
|
||||||
else:
|
else:
|
||||||
return False
|
return False
|
||||||
|
|
|
@ -11,7 +11,7 @@ class GLOBAL:
|
||||||
"""Declare global variables"""
|
"""Declare global variables"""
|
||||||
|
|
||||||
RUN_TIME = 0
|
RUN_TIME = 0
|
||||||
config = None
|
config = {'imgur_client_id':None, 'imgur_client_secret': None}
|
||||||
arguments = None
|
arguments = None
|
||||||
directory = None
|
directory = None
|
||||||
defaultConfigDirectory = Path.home() / "Bulk Downloader for Reddit"
|
defaultConfigDirectory = Path.home() / "Bulk Downloader for Reddit"
|
Loading…
Reference in a new issue