From 837281c3c6627041f4754aa3fbce17580cd4abc2 Mon Sep 17 00:00:00 2001 From: Ali Parlakci Date: Wed, 25 Jul 2018 13:40:06 +0300 Subject: [PATCH 1/4] Added verbose mode --- docs/COMMAND_LINE_ARGUMENTS.md | 5 ++-- script.py | 7 ++++- src/searcher.py | 55 +++++++++++++++++++++------------- src/tools.py | 11 +++---- 4 files changed, 50 insertions(+), 28 deletions(-) diff --git a/docs/COMMAND_LINE_ARGUMENTS.md b/docs/COMMAND_LINE_ARGUMENTS.md index 5c15e09..56b5372 100644 --- a/docs/COMMAND_LINE_ARGUMENTS.md +++ b/docs/COMMAND_LINE_ARGUMENTS.md @@ -10,13 +10,13 @@ usage: script.py [-h] [--directory DIRECTORY] [--link link] [--saved] [--subreddit SUBREDDIT [SUBREDDIT ...]] [--multireddit MULTIREDDIT] [--user redditor] [--search query] [--sort SORT TYPE] [--limit Limit] - [--time TIME_LIMIT] [--NoDownload] + [--time TIME_LIMIT] [--NoDownload] [--verbose] This program downloads media from reddit posts optional arguments: -h, --help show this help message and exit - --directory DIRECTORY + --directory DIRECTORY, -d DIRECTORY Specifies the directory where posts will be downloaded to --link link, -l link Get posts from link @@ -40,6 +40,7 @@ optional arguments: all --NoDownload Just gets the posts and store them in a file for downloading later + --verbose, -v Verbose Mode ``` # Examples diff --git a/script.py b/script.py index f5cac50..1ab10b4 100644 --- a/script.py +++ b/script.py @@ -62,7 +62,7 @@ def parseArguments(arguments=[]): description="This program downloads " \ "media from reddit " \ "posts") - parser.add_argument("--directory", + parser.add_argument("--directory","-d", help="Specifies the directory where posts will be " \ "downloaded to", metavar="DIRECTORY") @@ -144,6 +144,11 @@ def parseArguments(arguments=[]): action="store_true", default=False) + parser.add_argument("--verbose","-v", + help="Verbose Mode", + action="store_true", + default=False) + if arguments == []: return parser.parse_args() diff --git a/src/searcher.py b/src/searcher.py index d534437..e170cea 100644 --- a/src/searcher.py +++ b/src/searcher.py @@ -1,4 +1,5 @@ import os +import sys import random import socket import webbrowser @@ -306,6 +307,7 @@ def redditSearcher(posts,SINGLE_POST=False): allPosts = {} + print("GETTING POSTS") postsFile = createLogFile("POSTS") if SINGLE_POST: @@ -326,40 +328,53 @@ def redditSearcher(posts,SINGLE_POST=False): if result is not None: details = result orderCount += 1 - printSubmission(submission,subCount,orderCount) + if GLOBAL.arguments.verbose: + printSubmission(submission,subCount,orderCount) subList.append(details) postsFile.add({subCount:[details]}) else: - for submission in posts: - subCount += 1 + try: + for submission in posts: + subCount += 1 - try: - details = {'postId':submission.id, - 'postTitle':submission.title, - 'postSubmitter':str(submission.author), - 'postType':None, - 'postURL':submission.url, - 'postSubreddit':submission.subreddit.display_name} - except AttributeError: - continue + if subCount % 100 == 0 and not GLOBAL.arguments.verbose: + sys.stdout.write("• ") + sys.stdout.flush() - result = checkIfMatching(submission) + if subCount % 1000 == 0: + sys.stdout.write("\n") + sys.stdout.flush() - if result is not None: - details = result - orderCount += 1 - printSubmission(submission,subCount,orderCount) - subList.append(details) + try: + details = {'postId':submission.id, + 'postTitle':submission.title, + 'postSubmitter':str(submission.author), + 'postType':None, + 'postURL':submission.url, + 'postSubreddit':submission.subreddit.display_name} + except AttributeError: + continue - allPosts[subCount] = [details] + result = checkIfMatching(submission) + + if result is not None: + details = result + orderCount += 1 + if GLOBAL.arguments.verbose: + printSubmission(submission,subCount,orderCount) + subList.append(details) + + allPosts[subCount] = [details] + except KeyboardInterrupt: + print("\nKeyboardInterrupt",end="") postsFile.add(allPosts) if not len(subList) == 0: print( - "\nTotal of {} submissions found!\n"\ + "\n\nTotal of {} submissions found!\n"\ "{} GFYCATs, {} IMGURs, {} EROMEs, {} DIRECTs and {} SELF POSTS\n" .format( len(subList), diff --git a/src/tools.py b/src/tools.py index b33c3a8..9f2eeda 100644 --- a/src/tools.py +++ b/src/tools.py @@ -102,11 +102,12 @@ def printToFile(*args, **kwargs): if not path.exists(folderDirectory): makedirs(folderDirectory) - - with io.open( - folderDirectory / "CONSOLE_LOG.txt","a",encoding="utf-8" - ) as FILE: - print(*args, file=FILE, **kwargs) + + if not "file" in kwargs: + with io.open( + folderDirectory / "CONSOLE_LOG.txt","a",encoding="utf-8" + ) as FILE: + print(*args, file=FILE, **kwargs) def nameCorrector(string): """Swap strange characters from given string From 394b864d8612e38c5fed35b70de275de875dd671 Mon Sep 17 00:00:00 2001 From: Ali Parlakci Date: Wed, 25 Jul 2018 13:48:02 +0300 Subject: [PATCH 2/4] Updated FAQ --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index a17b9c9..973ee0f 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,12 @@ It should redirect to a page which shows your **imgur_client_id** and **imgur_cl \* Select **OAuth 2 authorization without a callback URL** first then select **Anonymous usage without user authorization** if it says *Authorization callback URL: required* ## FAQ +### What are the dots resemble when getting posts? +- Each dot means that 100 posts are scanned. + +### Getting posts is taking too long. +- You can press Ctrl+C to interrupt it and start downloading. + ### How downloaded files' names are formatted? - Images that are not belong to an album or self posts are formatted as **`[SUBMITTER NAME]_[POST TITLE]_[REDDIT ID]`**. You can use *reddit id* to go to post's reddit page by going to link **reddit.com/[REDDIT ID]** From 35d54d1eb17d1d982e601c882fd58acc0ef40054 Mon Sep 17 00:00:00 2001 From: Ali Parlakci Date: Wed, 25 Jul 2018 13:48:30 +0300 Subject: [PATCH 3/4] Stylize --- src/searcher.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/searcher.py b/src/searcher.py index e170cea..4d9b0d1 100644 --- a/src/searcher.py +++ b/src/searcher.py @@ -374,16 +374,10 @@ def redditSearcher(posts,SINGLE_POST=False): if not len(subList) == 0: print( - "\n\nTotal of {} submissions found!\n"\ - "{} GFYCATs, {} IMGURs, {} EROMEs, {} DIRECTs and {} SELF POSTS\n" - .format( - len(subList), - gfycatCount, - imgurCount, - eromeCount, - directCount, - selfCount - ) + f"\n\nTotal of {len(subList)} submissions found!\n"\ + f"{gfycatCount} GFYCATs, {imgurCount} IMGURs, " \ + f"{eromeCount} EROMEs, {directCount} DIRECTs " \ + f"and {selfCount} SELF POSTS" ) return subList else: From dad5669441a72df7d85f7b9d230f39053d0ad0ab Mon Sep 17 00:00:00 2001 From: Ali Parlakci Date: Wed, 25 Jul 2018 13:50:11 +0300 Subject: [PATCH 4/4] Typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 973ee0f..38a9e14 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ It should redirect to a page which shows your **imgur_client_id** and **imgur_cl \* Select **OAuth 2 authorization without a callback URL** first then select **Anonymous usage without user authorization** if it says *Authorization callback URL: required* ## FAQ -### What are the dots resemble when getting posts? +### What do the dots resemble when getting posts? - Each dot means that 100 posts are scanned. ### Getting posts is taking too long.