From af1f9fd3659be50c0a601079f7e809ae51d590f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20Parlak=C3=A7=C4=B1?= Date: Wed, 3 Jun 2020 18:10:25 +0300 Subject: [PATCH] v1.9.0 (#114) * IMGUR API is no longer used * --skip now accepts file types instead of domain * --skip-domain added * --no-download added * --no-dupe now supports YouTube * Duplicates of older posts will not be dowloaded if --no-dupe and --downloaded-posts options are given together * Invalid characters in MacOS and Linux platforms are removed from filenames * Bug fixes --- README.md | 24 +-- requirements.txt | 1 - script.py | 110 +++++-------- setup.py | 2 +- src/arguments.py | 15 +- src/config.py | 27 +--- src/downloaders/Imgur.py | 245 +++++++++++++++-------------- src/downloaders/downloaderUtils.py | 25 ++- src/downloaders/selfPost.py | 5 +- src/downloaders/youtube.py | 4 +- src/errors.py | 9 ++ src/searcher.py | 63 +++----- src/store.py | 6 +- src/utils.py | 39 +++-- 14 files changed, 277 insertions(+), 298 deletions(-) diff --git a/README.md b/README.md index af22ae6..1a62506 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,6 @@ OR, regardless of your operating system, you can fire up the program from the ** See the [Interpret from source code](docs/INTERPRET_FROM_SOURCE.md) page for more information. ## 🔨 Setting up the program -### 🖼 IMGUR API - -You need to create an imgur developer app in order API to work. Go to https://api.imgur.com/oauth2/addclient and login. - -IMGUR will redirect you to homepage instead of API form page. After you log in, open the above link manually. Fill the form in the link (It does not really matter what you fill it with. You can write www.google.com to the callback url) - -After you send the form, it will redirect you to a page where it shows your **imgur_client_id** and **imgur_client_secret**. Type in those values into program respectively. - ### 📽 ffmpeg Library Program needs **ffmpeg software** to add audio to some video files. However, installing it is **voluntary**. Although the program can still run with no errors without the ffmpeg library, some video files might have no sound. @@ -113,9 +105,14 @@ Example usage: **`--limit 500`** --- ## **`--skip`** -Takes a number of domains as a parameter to skip the posts from those domains. Use self to imply text posts. +Takes a number of file types as a parameter to skip the posts from those domains. Valid file types are `images`, `videos`, `gifs`, `self` + +Example usage: **`--skip self videos`** + +## **`--skip-domain`** +Takes a number of domains as a parameter to skip the posts from those domains. -Example usage: **`--skip v.redd.it youtube.com youtu.be self`** +Example usage: **`--skip v.redd.it youtube.com youtu.be`** ## **`--quit`** Automatically quits the application after it finishes. Otherwise, it will wait for an input to quit. @@ -167,8 +164,13 @@ Skips the same posts in different subreddits. Does not take any parameter. Example usage: **`--no-dupes`** +## **`--no-download`** +Quits the program without downloading the posts. Does not take any parameter + +Example usage: **`--no-download`** + ## **`--downloaded-posts`** -Takes a file directory as a parameter and skips the posts if it matches with the post IDs inside the file. It also saves the newly downloaded posts to the given file. Does not take any parameter. +Takes a file directory as a parameter and skips the posts if it matches with the post IDs inside the file. It also saves the newly downloaded posts to the given file. Example usage: **`--downloaded-posts D:\bdfr\ALL_POSTS.txt`** diff --git a/requirements.txt b/requirements.txt index cb6eeea..1a68e57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ bs4 requests praw -imgurpython youtube-dl \ No newline at end of file diff --git a/script.py b/script.py index b8a7dc4..cb2b484 100644 --- a/script.py +++ b/script.py @@ -24,7 +24,7 @@ from src.downloaders.selfPost import SelfPost from src.downloaders.vreddit import VReddit from src.downloaders.youtube import Youtube from src.downloaders.gifDeliveryNetwork import GifDeliveryNetwork -from src.errors import ImgurLimitError, NoSuitablePost, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, DomainInSkip, full_exc_info +from src.errors import ImgurLimitError, NoSuitablePost, FileAlreadyExistsError, ImgurLoginError, NotADownloadableLinkError, NoSuitablePost, InvalidJSONFile, FailedToDownload, TypeInSkip, DomainInSkip, AlbumNotDownloadedCompletely, full_exc_info from src.parser import LinkDesigner from src.searcher import getPosts from src.utils import (GLOBAL, createLogFile, nameCorrector, @@ -38,7 +38,7 @@ from src.store import Store __author__ = "Ali Parlakci" __license__ = "GPL" -__version__ = "1.8.0" +__version__ = "1.9.0" __maintainer__ = "Ali Parlakci" __email__ = "parlakciali@gmail.com" @@ -84,9 +84,6 @@ def isPostExists(POST,directory): def downloadPost(SUBMISSION,directory): - global lastRequestTime - lastRequestTime = 0 - downloaders = { "imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":SelfPost, "redgifs":Redgifs, "gifdeliverynetwork": GifDeliveryNetwork, @@ -95,55 +92,7 @@ def downloadPost(SUBMISSION,directory): print() if SUBMISSION['TYPE'] in downloaders: - - # WORKAROUND FOR IMGUR API LIMIT - if SUBMISSION['TYPE'] == "imgur": - - while int(time.time() - lastRequestTime) <= 2: - pass - - credit = Imgur.get_credits() - - IMGUR_RESET_TIME = credit['UserReset']-time.time() - USER_RESET = ("after " \ - + str(int(IMGUR_RESET_TIME/60)) \ - + " Minutes " \ - + str(int(IMGUR_RESET_TIME%60)) \ - + " Seconds") - - if credit['ClientRemaining'] < 25 or credit['UserRemaining'] < 25: - printCredit = {"noPrint":False} - else: - printCredit = {"noPrint":True} - - print( - "==> Client: {} - User: {} - Reset {}\n".format( - credit['ClientRemaining'], - credit['UserRemaining'], - USER_RESET - ),end="",**printCredit - ) - - if not (credit['UserRemaining'] == 0 or \ - credit['ClientRemaining'] == 0): - - """This block of code is needed for API workaround - """ - while int(time.time() - lastRequestTime) <= 2: - pass - - lastRequestTime = time.time() - - else: - if credit['UserRemaining'] == 0: - KEYWORD = "user" - elif credit['ClientRemaining'] == 0: - KEYWORD = "client" - - raise ImgurLimitError('{} LIMIT EXCEEDED\n'.format(KEYWORD.upper())) - downloaders[SUBMISSION['TYPE']] (directory,SUBMISSION) - else: raise NoSuitablePost @@ -154,8 +103,6 @@ def download(submissions): to download each one, catch errors, update the log files """ - global lastRequestTime - lastRequestTime = 0 downloadedCount = 0 duplicates = 0 @@ -164,7 +111,6 @@ def download(submissions): if GLOBAL.arguments.unsave: reddit = Reddit(GLOBAL.config['credentials']['reddit']).begin() - submissions = list(filter(lambda x: x['POSTID'] not in GLOBAL.downloadedPosts(), submissions)) subsLenght = len(submissions) for i in range(len(submissions)): @@ -177,13 +123,24 @@ def download(submissions): end="") print(f" – {submissions[i]['TYPE'].upper()}",end="",noPrint=True) - details = {**submissions[i], **{"TITLE": nameCorrector(submissions[i]['TITLE'])}} - directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**details) + directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**submissions[i]) + details = { + **submissions[i], + **{ + "TITLE": nameCorrector( + submissions[i]['TITLE'], + reference = str(directory) + + GLOBAL.config['filename'].format(**submissions[i]) + + ".ext" + ) + } + } + filename = GLOBAL.config['filename'].format(**details) if isPostExists(details,directory): print() print(directory) - print(GLOBAL.config['filename'].format(**details)) + print(filename) print("It already exists") duplicates += 1 continue @@ -227,7 +184,7 @@ def download(submissions): except NotADownloadableLinkError as exception: print( - "{class_name}: {info} See CONSOLE_LOG.txt for more information".format( + "{class_name}: {info}".format( class_name=exception.__class__.__name__,info=str(exception) ) ) @@ -238,27 +195,40 @@ def download(submissions): submissions[i] ]}) + except TypeInSkip: + print() + print(submissions[i]['CONTENTURL']) + print("Skipping post...") + except DomainInSkip: print() print(submissions[i]['CONTENTURL']) - print("Domain found in skip domains, skipping post...") + print("Skipping post...") except NoSuitablePost: print("No match found, skipping...") except FailedToDownload: print("Failed to download the posts, skipping...") + except AlbumNotDownloadedCompletely: + print("Album did not downloaded completely.") + FAILED_FILE.add({int(i+1):[ + "{class_name}: {info}".format( + class_name=exc.__class__.__name__,info=str(exc) + ), + submissions[i] + ]}) except Exception as exc: print( - "{class_name}: {info} See CONSOLE_LOG.txt for more information".format( + "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format( class_name=exc.__class__.__name__,info=str(exc) ) ) logging.error(sys.exc_info()[0].__name__, exc_info=full_exc_info(sys.exc_info())) - print(log_stream.getvalue(),noPrint=True) + print(GLOBAL.log_stream.getvalue(),noPrint=True) FAILED_FILE.add({int(i+1):[ "{class_name}: {info}".format( @@ -355,7 +325,7 @@ def main(): except Exception as exc: logging.error(sys.exc_info()[0].__name__, exc_info=full_exc_info(sys.exc_info())) - print(log_stream.getvalue(),noPrint=True) + print(GLOBAL.log_stream.getvalue(),noPrint=True) print(exc) sys.exit() @@ -363,12 +333,13 @@ def main(): print("I could not find any posts in that URL") sys.exit() - download(posts) + if GLOBAL.arguments.no_download: pass + else: download(posts) if __name__ == "__main__": - log_stream = StringIO() - logging.basicConfig(stream=log_stream, level=logging.INFO) + GLOBAL.log_stream = StringIO() + logging.basicConfig(stream=GLOBAL.log_stream, level=logging.INFO) try: VanillaPrint = print @@ -388,6 +359,7 @@ if __name__ == "__main__": GLOBAL.directory = Path("..\\") logging.error(sys.exc_info()[0].__name__, exc_info=full_exc_info(sys.exc_info())) - print(log_stream.getvalue()) + print(GLOBAL.log_stream.getvalue()) - if not GLOBAL.arguments.quit: input("\nPress enter to quit\n") \ No newline at end of file + if not GLOBAL.arguments.quit: input("\nPress enter to quit\n") + \ No newline at end of file diff --git a/setup.py b/setup.py index d269842..cb687bd 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ from script import __version__ options = { "build_exe": { "packages":[ - "idna","imgurpython", "praw", "requests", "multiprocessing" + "idna", "praw", "requests", "multiprocessing" ] } } diff --git a/src/arguments.py b/src/arguments.py index 642dbff..dd7c0e9 100644 --- a/src/arguments.py +++ b/src/arguments.py @@ -102,6 +102,13 @@ class Arguments: type=str) parser.add_argument("--skip", + nargs="+", + help="Skip posts with given type", + type=str, + choices=["images","videos","gifs","self"], + default=[]) + + parser.add_argument("--skip-domain", nargs="+", help="Skip posts with given domain", type=str, @@ -140,7 +147,13 @@ class Arguments: parser.add_argument("--downloaded-posts", help="Use a hash file to keep track of downloaded files", type=str - ) + ) + + parser.add_argument("--no-download", + action="store_true", + help="Just saved posts into a the POSTS.json file without downloading" + ) + if arguments == []: return parser.parse_args() diff --git a/src/config.py b/src/config.py index f6a0783..6c4c843 100644 --- a/src/config.py +++ b/src/config.py @@ -5,6 +5,7 @@ import random from src.reddit import Reddit from src.jsonHelper import JsonFile +from src.utils import nameCorrector class Config(): @@ -36,7 +37,7 @@ For example: {FLAIR}_{SUBREDDIT}_{REDDITOR} Existing filename template:""", None if "filename" not in self.file.read() else self.file.read()["filename"]) - filename = input(">> ").upper() + filename = nameCorrector(input(">> ").upper()) self.file.add({ "filename": filename }) @@ -68,7 +69,7 @@ For example: {REDDITOR}/{SUBREDDIT}/{FLAIR} Existing folder structure""", None if "folderpath" not in self.file.read() else self.file.read()["folderpath"]) - folderpath = input(">> ").strip("\\").strip("/").upper() + folderpath = nameCorrector(input(">> ").strip("\\").strip("/").upper()) self.file.add({ "folderpath": folderpath @@ -105,8 +106,6 @@ Existing default options:""", None if "options" not in self.file.read() else sel def _validateCredentials(self): """Read credentials from config.json file""" - keys = ['imgur_client_id', - 'imgur_client_secret'] try: content = self.file.read()["credentials"] except: @@ -119,25 +118,7 @@ Existing default options:""", None if "options" not in self.file.read() else sel pass else: Reddit().begin() - - if not all(content.get(key,False) for key in keys): - print( - "---Setting up the Imgur API---\n\n" \ - "Go to this URL and fill the form:\n" \ - "https://api.imgur.com/oauth2/addclient\n" \ - "Then, enter the client id and client secret here\n" \ - "Press Enter to open the link in the browser" - ) - input() - webbrowser.open("https://api.imgur.com/oauth2/addclient",new=2) - - for key in keys: - try: - if content[key] == "": - raise KeyError - except KeyError: - self.file.add({key:input("\t"+key+": ")}, - "credentials") + print() def setDefaultDirectory(self): diff --git a/src/downloaders/Imgur.py b/src/downloaders/Imgur.py index 246c609..0f82c28 100644 --- a/src/downloaders/Imgur.py +++ b/src/downloaders/Imgur.py @@ -1,137 +1,142 @@ +import urllib +import json import os +import time +import requests -import imgurpython - -from src.downloaders.downloaderUtils import getExtension, getFile -from src.errors import (AlbumNotDownloadedCompletely, FileAlreadyExistsError, - FileNameTooLong) from src.utils import GLOBAL, nameCorrector from src.utils import printToFile as print - +from src.downloaders.Direct import Direct +from src.downloaders.downloaderUtils import getFile +from src.errors import FileNotFoundError, FileAlreadyExistsError, AlbumNotDownloadedCompletely, ImageNotFound, ExtensionError, NotADownloadableLinkError, TypeInSkip class Imgur: - def __init__(self,directory,post): - self.imgurClient = self.initImgur() - imgurID = self.getId(post['CONTENTURL']) - content = self.getLink(imgurID) + IMGUR_IMAGE_DOMAIN = "https://i.imgur.com/" - if not os.path.exists(directory): os.makedirs(directory) + def __init__(self,directory, post): - if content['type'] == 'image': + link = post['CONTENTURL'] - try: - post['MEDIAURL'] = content['object'].mp4 - except AttributeError: - post['MEDIAURL'] = content['object'].link + if link.endswith(".gifv"): + link = link.replace(".gifv",".mp4") + Direct(directory, {**post, 'CONTENTURL': link}) + return None - post['EXTENSION'] = getExtension(post['MEDIAURL']) + self.rawData = self.getData(link) - filename = GLOBAL.config['filename'].format(**post)+post["EXTENSION"] - shortFilename = post['POSTID']+post['EXTENSION'] - - getFile(filename,shortFilename,directory,post['MEDIAURL']) - - elif content['type'] == 'album': - images = content['object'].images - imagesLenght = len(images) - howManyDownloaded = imagesLenght - duplicates = 0 - - filename = GLOBAL.config['filename'].format(**post) - - print(filename) - - folderDir = directory / filename - - try: - if not os.path.exists(folderDir): - os.makedirs(folderDir) - except FileNotFoundError: - folderDir = directory / post['POSTID'] - os.makedirs(folderDir) - - for i in range(imagesLenght): - try: - imageURL = images[i]['mp4'] - except KeyError: - imageURL = images[i]['link'] - - images[i]['Ext'] = getExtension(imageURL) - - filename = (str(i+1) - + "_" - + nameCorrector(str(images[i]['title'])) - + "_" - + images[i]['id']) - - shortFilename = (str(i+1) + "_" + images[i]['id']) - - print("\n ({}/{})".format(i+1,imagesLenght)) - - try: - getFile(filename,shortFilename,folderDir,imageURL,indent=2) - print() - except FileAlreadyExistsError: - print(" The file already exists" + " "*10,end="\n\n") - duplicates += 1 - howManyDownloaded -= 1 - - except Exception as exception: - print("\n Could not get the file") - print( - " " - + "{class_name}: {info}".format( - class_name=exception.__class__.__name__, - info=str(exception) - ) - + "\n" - ) - howManyDownloaded -= 1 - - if duplicates == imagesLenght: - raise FileAlreadyExistsError - elif howManyDownloaded + duplicates < imagesLenght: - raise AlbumNotDownloadedCompletely( - "Album Not Downloaded Completely" - ) - - @staticmethod - def initImgur(): - """Initialize imgur api""" - - config = GLOBAL.config - return imgurpython.ImgurClient( - config["credentials"]['imgur_client_id'], - config["credentials"]['imgur_client_secret'] - ) - def getId(self,submissionURL): - """Extract imgur post id - and determine if its a single image or album - """ - - if submissionURL[-1] == "/": - submissionURL = submissionURL[:-1] - - if "a/" in submissionURL or "gallery/" in submissionURL: - albumId = submissionURL.split("/")[-1] - return {'id':albumId, 'type':'album'} + self.directory = directory + self.post = post + if self.isAlbum: + if self.rawData["album_images"]["count"] != 1: + self.downloadAlbum(self.rawData["album_images"]) + else: + self.download(self.rawData["album_images"]["images"][0]) else: - url = submissionURL.replace('.','/').split('/') - imageId = url[url.index('com')+1] - return {'id':imageId, 'type':'image'} + self.download(self.rawData) - def getLink(self,identity): - """Request imgur object from imgur api - """ + def downloadAlbum(self, images): + folderName = GLOBAL.config['filename'].format(**self.post) + folderDir = self.directory / folderName + + imagesLenght = images["count"] + howManyDownloaded = 0 + duplicates = 0 + + try: + if not os.path.exists(folderDir): + os.makedirs(folderDir) + except FileNotFoundError: + folderDir = self.directory / self.post['POSTID'] + os.makedirs(folderDir) + + print(folderName) + + for i in range(imagesLenght): + + extension = self.validateExtension(images["images"][i]["ext"]) + + imageURL = self.IMGUR_IMAGE_DOMAIN + images["images"][i]["hash"] + extension + + filename = "_".join([ + str(i+1), nameCorrector(images["images"][i]['title']), images["images"][i]['hash'] + ]) + extension + shortFilename = str(i+1) + "_" + images["images"][i]['hash'] + + print("\n ({}/{})".format(i+1,imagesLenght)) + + try: + getFile(filename,shortFilename,folderDir,imageURL,indent=2) + howManyDownloaded += 1 + print() + + except FileAlreadyExistsError: + print(" The file already exists" + " "*10,end="\n\n") + duplicates += 1 + + except TypeInSkip: + print(" Skipping...") + howManyDownloaded += 1 + + except Exception as exception: + print("\n Could not get the file") + print( + " " + + "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format( + class_name=exception.__class__.__name__, + info=str(exception) + ) + + "\n" + ) + print(GLOBAL.log_stream.getvalue(),noPrint=True) + + if duplicates == imagesLenght: + raise FileAlreadyExistsError + elif howManyDownloaded + duplicates < imagesLenght: + raise AlbumNotDownloadedCompletely( + "Album Not Downloaded Completely" + ) + + def download(self, image): + extension = self.validateExtension(image["ext"]) + imageURL = self.IMGUR_IMAGE_DOMAIN + image["hash"] + extension + + filename = GLOBAL.config['filename'].format(**self.post) + extension + shortFilename = self.post['POSTID']+extension + + getFile(filename,shortFilename,self.directory,imageURL) + + @property + def isAlbum(self): + return "album_images" in self.rawData + + @staticmethod + def getData(link): + + cookies = {"over18": "1"} + res = requests.get(link, cookies=cookies) + if res.status_code != 200: raise ImageNotFound(f"Server responded with {res.status_code} to {link}") + pageSource = requests.get(link, cookies=cookies).text + + STARTING_STRING = "image : " + ENDING_STRING = "group :" + + STARTING_STRING_LENGHT = len(STARTING_STRING) + try: + startIndex = pageSource.index(STARTING_STRING) + STARTING_STRING_LENGHT + endIndex = pageSource.index(ENDING_STRING) + except ValueError: + raise NotADownloadableLinkError(f"Could not read the page source on {link}") + + data = pageSource[startIndex:endIndex].strip()[:-1] + + return json.loads(data) - if identity['type'] == 'image': - return {'object':self.imgurClient.get_image(identity['id']), - 'type':'image'} - elif identity['type'] == 'album': - return {'object':self.imgurClient.get_album(identity['id']), - 'type':'album'} @staticmethod - def get_credits(): - return Imgur.initImgur().get_credits() + def validateExtension(string): + POSSIBLE_EXTENSIONS = [".jpg", ".png", ".mp4", ".gif"] + + for extension in POSSIBLE_EXTENSIONS: + if extension in string: return extension + else: raise ExtensionError(f"\"{string}\" is not recognized as a valid extension.") diff --git a/src/downloaders/downloaderUtils.py b/src/downloaders/downloaderUtils.py index d194aab..bb77f93 100644 --- a/src/downloaders/downloaderUtils.py +++ b/src/downloaders/downloaderUtils.py @@ -8,7 +8,7 @@ import hashlib from src.utils import nameCorrector, GLOBAL from src.utils import printToFile as print -from src.errors import FileAlreadyExistsError, FileNameTooLong, FailedToDownload, DomainInSkip +from src.errors import FileAlreadyExistsError, FileNameTooLong, FailedToDownload, TypeInSkip, DomainInSkip def dlProgress(count, blockSize, totalSize): """Function for writing download progress to console @@ -37,7 +37,18 @@ def getExtension(link): def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False): - if any(domain in imageURL for domain in GLOBAL.arguments.skip): + FORMATS = { + "videos": [".mp4", ".webm"], + "images": [".jpg",".jpeg",".png",".bmp"], + "gifs": [".gif"] + } + + for type in GLOBAL.arguments.skip: + for extension in FORMATS[type]: + if extension in filename: + raise TypeInSkip + + if any(domain in imageURL for domain in GLOBAL.arguments.skip_domain): raise DomainInSkip headers = [ @@ -52,13 +63,13 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False): ("Connection", "keep-alive") ] + if not os.path.exists(folderDir): os.makedirs(folderDir) + opener = urllib.request.build_opener() if not "imgur" in imageURL: opener.addheaders = headers urllib.request.install_opener(opener) - filename = nameCorrector(filename) - if not silent: print(" "*indent + str(folderDir), " "*indent + str(filename), sep="\n") @@ -74,12 +85,12 @@ def getFile(filename,shortFilename,folderDir,imageURL,indent=0, silent=False): tempDir, reporthook=dlProgress) + fileHash = createHash(tempDir) if GLOBAL.arguments.no_dupes: - fileHash = createHash(tempDir) - if fileHash in GLOBAL.hashList: + if fileHash in GLOBAL.downloadedPosts(): os.remove(tempDir) raise FileAlreadyExistsError - GLOBAL.hashList.add(fileHash) + GLOBAL.downloadedPosts.add(fileHash) os.rename(tempDir,fileDir) if not silent: print(" "*indent+"Downloaded"+" "*10) diff --git a/src/downloaders/selfPost.py b/src/downloaders/selfPost.py index aa8f709..9fee2f0 100644 --- a/src/downloaders/selfPost.py +++ b/src/downloaders/selfPost.py @@ -2,7 +2,7 @@ import io import os from pathlib import Path -from src.errors import FileAlreadyExistsError +from src.errors import FileAlreadyExistsError, TypeInSkip from src.utils import GLOBAL VanillaPrint = print @@ -10,6 +10,9 @@ from src.utils import printToFile as print class SelfPost: def __init__(self,directory,post): + + if "self" in GLOBAL.arguments.skip: raise TypeInSkip + if not os.path.exists(directory): os.makedirs(directory) filename = GLOBAL.config['filename'].format(**post) diff --git a/src/downloaders/youtube.py b/src/downloaders/youtube.py index 6207fc8..46cb37c 100644 --- a/src/downloaders/youtube.py +++ b/src/downloaders/youtube.py @@ -36,10 +36,10 @@ class Youtube: fileHash = createHash(location) except FileNotFoundError: return None - if fileHash in GLOBAL.hashList: + if fileHash in GLOBAL.downloadedPosts(): os.remove(location) raise FileAlreadyExistsError - GLOBAL.hashList.add(fileHash) + GLOBAL.downloadedPosts.add(fileHash) @staticmethod def _hook(d): diff --git a/src/errors.py b/src/errors.py index c3a0f99..0877e62 100644 --- a/src/errors.py +++ b/src/errors.py @@ -99,5 +99,14 @@ class InvalidJSONFile(Exception): class FailedToDownload(Exception): pass +class TypeInSkip(Exception): + pass + class DomainInSkip(Exception): + pass + +class ImageNotFound(Exception): + pass + +class ExtensionError(Exception): pass \ No newline at end of file diff --git a/src/searcher.py b/src/searcher.py index ce39ee4..93bf287 100644 --- a/src/searcher.py +++ b/src/searcher.py @@ -201,7 +201,7 @@ def extractDetails(posts,SINGLE_POST=False): """ postList = [] - postCount = 0 + postCount = 1 allPosts = {} @@ -227,18 +227,17 @@ def extractDetails(posts,SINGLE_POST=False): except AttributeError: pass - result = matchWithDownloader(submission) + if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): + result = matchWithDownloader(submission) - if result is not None: - details = {**details, **result} - postList.append(details) - - postsFile.add({postCount:details}) + if result is not None: + details = {**details, **result} + postList.append(details) + postsFile.add({postCount:details}) else: try: for submission in posts: - postCount += 1 if postCount % 100 == 0: sys.stdout.write("• ") @@ -264,13 +263,18 @@ def extractDetails(posts,SINGLE_POST=False): except AttributeError: continue - result = matchWithDownloader(submission) + if details['POSTID'] in GLOBAL.downloadedPosts(): continue - if result is not None: - details = {**details, **result} - postList.append(details) + if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): + result = matchWithDownloader(submission) - allPosts[postCount] = details + if result is not None: + details = {**details, **result} + postList.append(details) + + allPosts[postCount] = details + postCount += 1 + except KeyboardInterrupt: print("\nKeyboardInterrupt",noPrint=True) @@ -284,6 +288,11 @@ def extractDetails(posts,SINGLE_POST=False): def matchWithDownloader(submission): + directLink = extractDirectLink(submission.url) + if directLink: + return {'TYPE': 'direct', + 'CONTENTURL': directLink} + if 'v.redd.it' in submission.domain: bitrates = ["DASH_1080","DASH_720","DASH_600", \ "DASH_480","DASH_360","DASH_240"] @@ -291,7 +300,7 @@ def matchWithDownloader(submission): for bitrate in bitrates: videoURL = submission.url+"/"+bitrate - try: + try: responseCode = urllib.request.urlopen(videoURL).getcode() except urllib.error.HTTPError: responseCode = 0 @@ -327,12 +336,6 @@ def matchWithDownloader(submission): return {'TYPE': 'self', 'CONTENT': submission.selftext} - try: - return {'TYPE': 'direct', - 'CONTENTURL': extractDirectLink(submission.url)} - except DirectLinkNotFound: - return None - def extractDirectLink(URL): """Check if link is a direct image link. If so, return URL, @@ -346,26 +349,8 @@ def extractDirectLink(URL): if "i.reddituploads.com" in URL: return URL - elif "v.redd.it" in URL: - bitrates = ["DASH_1080","DASH_720","DASH_600", \ - "DASH_480","DASH_360","DASH_240"] - - for bitrate in bitrates: - videoURL = URL+"/"+bitrate - - try: - responseCode = urllib.request.urlopen(videoURL).getcode() - except urllib.error.HTTPError: - responseCode = 0 - - if responseCode == 200: - return videoURL - - else: - raise DirectLinkNotFound - for extension in imageTypes: if extension in URL.split("/")[-1]: return URL else: - raise DirectLinkNotFound + return None diff --git a/src/store.py b/src/store.py index e406ade..9da77ce 100644 --- a/src/store.py +++ b/src/store.py @@ -17,8 +17,8 @@ class Store: def __call__(self): return self.list - def add(self, filehash): - self.list.append(filehash) + def add(self, data): + self.list.append(data) if self.directory: with open(self.directory, 'a') as f: - f.write("{filehash}\n".format(filehash=filehash)) \ No newline at end of file + f.write("{data}\n".format(data=data)) diff --git a/src/utils.py b/src/utils.py index 85799ba..95d44d5 100644 --- a/src/utils.py +++ b/src/utils.py @@ -18,10 +18,11 @@ class GLOBAL: configDirectory = "" reddit_client_id = "U-6gk4ZCh3IeNQ" reddit_client_secret = "7CZHY6AmKweZME5s50SfDGylaPg" - hashList = set() downloadedPosts = lambda: [] printVanilla = print + log_stream= None + def createLogFile(TITLE): """Create a log file with given name inside a folder time stampt in its name and @@ -63,34 +64,32 @@ def printToFile(*args, noPrint=False,**kwargs): ) as FILE: print(*args, file=FILE, **kwargs) -def nameCorrector(string): +def nameCorrector(string,reference=None): """Swap strange characters from given string with underscore (_) and shorten it. Return the string """ - stringLenght = len(string) - if stringLenght > 200: - string = string[:200] - stringLenght = len(string) - spacesRemoved = [] + LIMIT = 247 - for b in range(stringLenght): - if string[b] == " ": - spacesRemoved.append("_") - else: - spacesRemoved.append(string[b]) - - string = ''.join(spacesRemoved) + stringLength = len(string) + + if reference: + referenceLenght = len(reference) + totalLenght = referenceLenght + else: + totalLenght = stringLength + + if totalLenght > LIMIT: + limit = LIMIT - referenceLenght + string = string[:limit-1] + + string = string.replace(" ", "_") if len(string.split('\n')) > 1: string = "".join(string.split('\n')) - BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','#'] - - if any(x in string for x in BAD_CHARS): - for char in string: - if char in BAD_CHARS: - string = string.replace(char,"_") + BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','#', '.', '@' ,'“', '’', '\'', '!'] + string = "".join([i if i not in BAD_CHARS else "_" for i in string]) return string \ No newline at end of file