Merge pull request #41 from aliparlakci/changePostNames
Add submitter to file names
This commit is contained in:
commit
7a68ff3efa
|
@ -12,7 +12,6 @@ This program downloads imgur, gfycat and direct image and video links of saved p
|
||||||
- Puts every post to its subreddit's folder
|
- Puts every post to its subreddit's folder
|
||||||
- Saves a reusable copy of posts' details that are found so that they can be re-downloaded again
|
- Saves a reusable copy of posts' details that are found so that they can be re-downloaded again
|
||||||
- Logs failed ones in a file to so that you can try to download them later
|
- Logs failed ones in a file to so that you can try to download them later
|
||||||
- Can run with double-clicking on Windows
|
|
||||||
|
|
||||||
## [Download the latest release](https://github.com/aliparlakci/bulk-downloader-for-reddit/releases/latest)
|
## [Download the latest release](https://github.com/aliparlakci/bulk-downloader-for-reddit/releases/latest)
|
||||||
|
|
||||||
|
|
35
script.py
35
script.py
|
@ -423,21 +423,36 @@ def postFromLog(fileName):
|
||||||
|
|
||||||
return posts
|
return posts
|
||||||
|
|
||||||
def postExists(POST):
|
def isPostExists(POST):
|
||||||
"""Figure out a file's name and checks if the file already exists"""
|
"""Figure out a file's name and checks if the file already exists"""
|
||||||
|
|
||||||
title = nameCorrector(POST['postTitle'])
|
title = nameCorrector(POST['postTitle'])
|
||||||
FILENAME = title + "_" + POST['postId']
|
|
||||||
PATH = GLOBAL.directory / POST["postSubreddit"]
|
PATH = GLOBAL.directory / POST["postSubreddit"]
|
||||||
|
|
||||||
possibleExtensions = [".jpg",".png",".mp4",".gif",".webm",".md"]
|
possibleExtensions = [".jpg",".png",".mp4",".gif",".webm",".md"]
|
||||||
|
|
||||||
for i in range(2):
|
for extension in possibleExtensions:
|
||||||
for extension in possibleExtensions:
|
|
||||||
FILE_PATH = PATH / (FILENAME+extension)
|
OLD_FILE_PATH = PATH / (
|
||||||
if FILE_PATH.exists():
|
title
|
||||||
return True
|
+ "_" + POST['postId']
|
||||||
else:
|
+ extension
|
||||||
FILENAME = POST['postId']
|
)
|
||||||
|
FILE_PATH = PATH / (
|
||||||
|
POST["postSubmitter"]
|
||||||
|
+ "_" + title
|
||||||
|
+ "_" + POST['postId']
|
||||||
|
+ extension
|
||||||
|
)
|
||||||
|
|
||||||
|
SHORT_FILE_PATH = PATH / (POST['postId']+extension)
|
||||||
|
|
||||||
|
if OLD_FILE_PATH.exists() or \
|
||||||
|
FILE_PATH.exists() or \
|
||||||
|
SHORT_FILE_PATH.exists():
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
else:
|
else:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@ -523,7 +538,7 @@ def download(submissions):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
if postExists(submissions[i]):
|
if isPostExists(submissions[i]):
|
||||||
print(submissions[i]['postType'].upper())
|
print(submissions[i]['postType'].upper())
|
||||||
print("It already exists")
|
print("It already exists")
|
||||||
duplicates += 1
|
duplicates += 1
|
||||||
|
|
|
@ -87,13 +87,14 @@ class Erome:
|
||||||
extension = getExtension(IMAGES[0])
|
extension = getExtension(IMAGES[0])
|
||||||
|
|
||||||
title = nameCorrector(post['postTitle'])
|
title = nameCorrector(post['postTitle'])
|
||||||
print(title+"_" +post['postId']+extension)
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+extension)
|
||||||
|
|
||||||
fileDir = title + "_" + post['postId'] + extension
|
fileDir = directory / (
|
||||||
fileDir = directory / fileDir
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+extension
|
||||||
|
)
|
||||||
tempDir = title + "_" + post['postId'] + '.tmp'
|
tempDir = directory / (
|
||||||
tempDir = directory / tempDir
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
imageURL = "https:" + IMAGES[0]
|
imageURL = "https:" + IMAGES[0]
|
||||||
|
|
||||||
|
@ -106,9 +107,11 @@ class Erome:
|
||||||
|
|
||||||
else:
|
else:
|
||||||
title = nameCorrector(post['postTitle'])
|
title = nameCorrector(post['postTitle'])
|
||||||
print(title+"_"+post['postId'],end="\n\n")
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
||||||
|
|
||||||
folderDir = directory / (title+"_"+post['postId'])
|
folderDir = directory / (
|
||||||
|
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if not os.path.exists(folderDir):
|
if not os.path.exists(folderDir):
|
||||||
|
@ -220,13 +223,22 @@ class Imgur:
|
||||||
post['postExt'] = getExtension(post['mediaURL'])
|
post['postExt'] = getExtension(post['mediaURL'])
|
||||||
|
|
||||||
title = nameCorrector(post['postTitle'])
|
title = nameCorrector(post['postTitle'])
|
||||||
print(title+"_" +post['postId']+post['postExt'])
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+post['postExt'])
|
||||||
|
|
||||||
fileDir = title + "_" + post['postId'] + post['postExt']
|
fileDir = directory / (
|
||||||
fileDir = directory / fileDir
|
post["postSubmitter"]
|
||||||
|
+ "_" + title
|
||||||
|
+ "_" + post['postId']
|
||||||
|
+ post['postExt']
|
||||||
|
)
|
||||||
|
|
||||||
|
tempDir = directory / (
|
||||||
|
post["postSubmitter"]
|
||||||
|
+ "_" + title
|
||||||
|
+ "_" + post['postId']
|
||||||
|
+ ".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
tempDir = title + "_" + post['postId'] + '.tmp'
|
|
||||||
tempDir = directory / tempDir
|
|
||||||
try:
|
try:
|
||||||
getFile(fileDir,tempDir,post['mediaURL'])
|
getFile(fileDir,tempDir,post['mediaURL'])
|
||||||
except FileNameTooLong:
|
except FileNameTooLong:
|
||||||
|
@ -242,9 +254,11 @@ class Imgur:
|
||||||
duplicates = 0
|
duplicates = 0
|
||||||
|
|
||||||
title = nameCorrector(post['postTitle'])
|
title = nameCorrector(post['postTitle'])
|
||||||
print(title+"_"+post['postId'],end="\n\n")
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
||||||
|
|
||||||
folderDir = directory / (title+"_"+post['postId'])
|
folderDir = directory / (
|
||||||
|
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if not os.path.exists(folderDir):
|
if not os.path.exists(folderDir):
|
||||||
|
@ -369,10 +383,15 @@ class Gfycat:
|
||||||
|
|
||||||
if not os.path.exists(directory): os.makedirs(directory)
|
if not os.path.exists(directory): os.makedirs(directory)
|
||||||
title = nameCorrector(POST['postTitle'])
|
title = nameCorrector(POST['postTitle'])
|
||||||
print(title+"_"+POST['postId']+POST['postExt'])
|
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||||
|
|
||||||
|
fileDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||||
|
)
|
||||||
|
tempDir = directory / (
|
||||||
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
fileDir = directory / (title+"_"+POST['postId']+POST['postExt'])
|
|
||||||
tempDir = directory / (title+"_"+POST['postId']+".tmp")
|
|
||||||
try:
|
try:
|
||||||
getFile(fileDir,tempDir,POST['mediaURL'])
|
getFile(fileDir,tempDir,POST['mediaURL'])
|
||||||
except FileNameTooLong:
|
except FileNameTooLong:
|
||||||
|
@ -418,13 +437,14 @@ class Direct:
|
||||||
POST['postExt'] = getExtension(POST['postURL'])
|
POST['postExt'] = getExtension(POST['postURL'])
|
||||||
if not os.path.exists(directory): os.makedirs(directory)
|
if not os.path.exists(directory): os.makedirs(directory)
|
||||||
title = nameCorrector(POST['postTitle'])
|
title = nameCorrector(POST['postTitle'])
|
||||||
print(title+"_"+POST['postId']+POST['postExt'])
|
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||||
|
|
||||||
fileDir = title+"_"+POST['postId']+POST['postExt']
|
fileDir = directory / (
|
||||||
fileDir = directory / fileDir
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||||
|
)
|
||||||
tempDir = title+"_"+POST['postId']+".tmp"
|
tempDir = directory / (
|
||||||
tempDir = directory / tempDir
|
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
getFile(fileDir,tempDir,POST['postURL'])
|
getFile(fileDir,tempDir,POST['postURL'])
|
||||||
|
@ -439,10 +459,11 @@ class Self:
|
||||||
if not os.path.exists(directory): os.makedirs(directory)
|
if not os.path.exists(directory): os.makedirs(directory)
|
||||||
|
|
||||||
title = nameCorrector(post['postTitle'])
|
title = nameCorrector(post['postTitle'])
|
||||||
print(title+"_"+post['postId']+".md")
|
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+".md")
|
||||||
|
|
||||||
fileDir = title+"_"+post['postId']+".md"
|
fileDir = directory / (
|
||||||
fileDir = directory / fileDir
|
post["postSubmitter"]+"_"+title+"_"+post['postId']+".md"
|
||||||
|
)
|
||||||
|
|
||||||
if Path.is_file(fileDir):
|
if Path.is_file(fileDir):
|
||||||
raise FileAlreadyExistsError
|
raise FileAlreadyExistsError
|
||||||
|
@ -465,7 +486,11 @@ class Self:
|
||||||
+ ")\n"
|
+ ")\n"
|
||||||
+ post["postContent"]
|
+ post["postContent"]
|
||||||
+ "\n\n---\n\n"
|
+ "\n\n---\n\n"
|
||||||
+ "submitted by [u/"
|
+ "submitted to [r/"
|
||||||
|
+ post["postSubreddit"]
|
||||||
|
+ "](https://www.reddit.com/r/"
|
||||||
|
+ post["postSubreddit"]
|
||||||
|
+ ") by [u/"
|
||||||
+ post["postSubmitter"]
|
+ post["postSubmitter"]
|
||||||
+ "](https://www.reddit.com/user/"
|
+ "](https://www.reddit.com/user/"
|
||||||
+ post["postSubmitter"]
|
+ post["postSubmitter"]
|
||||||
|
|
112
src/searcher.py
112
src/searcher.py
|
@ -14,60 +14,62 @@ from src.errors import (NoMatchingSubmissionFound, NoPrawSupport,
|
||||||
|
|
||||||
print = printToFile
|
print = printToFile
|
||||||
|
|
||||||
class GetAuth:
|
|
||||||
def __init__(self,redditInstance,port):
|
|
||||||
self.redditInstance = redditInstance
|
|
||||||
self.PORT = int(port)
|
|
||||||
|
|
||||||
def recieve_connection(self):
|
|
||||||
"""Wait for and then return a connected socket..
|
|
||||||
Opens a TCP connection on port 8080, and waits for a single client.
|
|
||||||
"""
|
|
||||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
||||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
||||||
server.bind(('localhost', self.PORT))
|
|
||||||
server.listen(1)
|
|
||||||
client = server.accept()[0]
|
|
||||||
server.close()
|
|
||||||
return client
|
|
||||||
|
|
||||||
def send_message(self, message):
|
|
||||||
"""Send message to client and close the connection."""
|
|
||||||
self.client.send('HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8'))
|
|
||||||
self.client.close()
|
|
||||||
|
|
||||||
def getRefreshToken(self,*scopes):
|
|
||||||
state = str(random.randint(0, 65000))
|
|
||||||
url = self.redditInstance.auth.url(scopes, state, 'permanent')
|
|
||||||
print("Go to this URL and login to reddit:\n\n",url)
|
|
||||||
webbrowser.open(url,new=2)
|
|
||||||
|
|
||||||
self.client = self.recieve_connection()
|
|
||||||
data = self.client.recv(1024).decode('utf-8')
|
|
||||||
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
|
|
||||||
params = {
|
|
||||||
key: value for (key, value) in [token.split('=') \
|
|
||||||
for token in param_tokens]
|
|
||||||
}
|
|
||||||
if state != params['state']:
|
|
||||||
self.send_message(
|
|
||||||
client, 'State mismatch. Expected: {} Received: {}'
|
|
||||||
.format(state, params['state'])
|
|
||||||
)
|
|
||||||
raise RedditLoginFailed
|
|
||||||
elif 'error' in params:
|
|
||||||
self.send_message(client, params['error'])
|
|
||||||
raise RedditLoginFailed
|
|
||||||
|
|
||||||
refresh_token = self.redditInstance.auth.authorize(params['code'])
|
|
||||||
self.send_message(
|
|
||||||
"<script>" \
|
|
||||||
"alert(\"You can go back to terminal window now.\");" \
|
|
||||||
"</script>"
|
|
||||||
)
|
|
||||||
return (self.redditInstance,refresh_token)
|
|
||||||
|
|
||||||
def beginPraw(config,user_agent = str(socket.gethostname())):
|
def beginPraw(config,user_agent = str(socket.gethostname())):
|
||||||
|
class GetAuth:
|
||||||
|
def __init__(self,redditInstance,port):
|
||||||
|
self.redditInstance = redditInstance
|
||||||
|
self.PORT = int(port)
|
||||||
|
|
||||||
|
def recieve_connection(self):
|
||||||
|
"""Wait for and then return a connected socket..
|
||||||
|
Opens a TCP connection on port 8080, and waits for a single client.
|
||||||
|
"""
|
||||||
|
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
server.bind(('localhost', self.PORT))
|
||||||
|
server.listen(1)
|
||||||
|
client = server.accept()[0]
|
||||||
|
server.close()
|
||||||
|
return client
|
||||||
|
|
||||||
|
def send_message(self, message):
|
||||||
|
"""Send message to client and close the connection."""
|
||||||
|
self.client.send(
|
||||||
|
'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8')
|
||||||
|
)
|
||||||
|
self.client.close()
|
||||||
|
|
||||||
|
def getRefreshToken(self,*scopes):
|
||||||
|
state = str(random.randint(0, 65000))
|
||||||
|
url = self.redditInstance.auth.url(scopes, state, 'permanent')
|
||||||
|
print("Go to this URL and login to reddit:\n\n",url)
|
||||||
|
webbrowser.open(url,new=2)
|
||||||
|
|
||||||
|
self.client = self.recieve_connection()
|
||||||
|
data = self.client.recv(1024).decode('utf-8')
|
||||||
|
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
|
||||||
|
params = {
|
||||||
|
key: value for (key, value) in [token.split('=') \
|
||||||
|
for token in param_tokens]
|
||||||
|
}
|
||||||
|
if state != params['state']:
|
||||||
|
self.send_message(
|
||||||
|
client, 'State mismatch. Expected: {} Received: {}'
|
||||||
|
.format(state, params['state'])
|
||||||
|
)
|
||||||
|
raise RedditLoginFailed
|
||||||
|
elif 'error' in params:
|
||||||
|
self.send_message(client, params['error'])
|
||||||
|
raise RedditLoginFailed
|
||||||
|
|
||||||
|
refresh_token = self.redditInstance.auth.authorize(params['code'])
|
||||||
|
self.send_message(
|
||||||
|
"<script>" \
|
||||||
|
"alert(\"You can go back to terminal window now.\");" \
|
||||||
|
"</script>"
|
||||||
|
)
|
||||||
|
return (self.redditInstance,refresh_token)
|
||||||
|
|
||||||
"""Start reddit instance"""
|
"""Start reddit instance"""
|
||||||
|
|
||||||
scopes = ['identity','history','read']
|
scopes = ['identity','history','read']
|
||||||
|
@ -245,8 +247,6 @@ def getPosts(args):
|
||||||
raise MultiredditNotFound
|
raise MultiredditNotFound
|
||||||
|
|
||||||
elif "submitted" in args:
|
elif "submitted" in args:
|
||||||
# TODO
|
|
||||||
# USE REDDIT.USER.ME() INSTEAD WHEN "ME" PASSED AS A --USER
|
|
||||||
print (
|
print (
|
||||||
"submitted posts of {user}\nsort: {sort}\n" \
|
"submitted posts of {user}\nsort: {sort}\n" \
|
||||||
"time: {time}\nlimit: {limit}\n".format(
|
"time: {time}\nlimit: {limit}\n".format(
|
||||||
|
@ -263,8 +263,6 @@ def getPosts(args):
|
||||||
)
|
)
|
||||||
|
|
||||||
elif "upvoted" in args:
|
elif "upvoted" in args:
|
||||||
# TODO
|
|
||||||
# USE REDDIT.USER.ME() INSTEAD WHEN "ME" PASSED AS A --USER
|
|
||||||
print (
|
print (
|
||||||
"upvoted posts of {user}\nlimit: {limit}\n".format(
|
"upvoted posts of {user}\nlimit: {limit}\n".format(
|
||||||
user=args["user"],
|
user=args["user"],
|
||||||
|
|
Loading…
Reference in a new issue