Merge pull request #41 from aliparlakci/changePostNames
Add submitter to file names
This commit is contained in:
Ali Parlakçı
GitHub
commit
7a68ff3efa
|
|
@ -12,7 +12,6 @@ This program downloads imgur, gfycat and direct image and video links of saved p
|
|||
- Puts every post to its subreddit's folder
|
||||
- Saves a reusable copy of posts' details that are found so that they can be re-downloaded again
|
||||
- Logs failed ones in a file to so that you can try to download them later
|
||||
- Can run with double-clicking on Windows
|
||||
|
||||
## [Download the latest release](https://github.com/aliparlakci/bulk-downloader-for-reddit/releases/latest)
|
||||
|
||||
|
|
|
|||
35
script.py
35
script.py
|
|
@ -423,21 +423,36 @@ def postFromLog(fileName):
|
|||
|
||||
return posts
|
||||
|
||||
def postExists(POST):
|
||||
def isPostExists(POST):
|
||||
"""Figure out a file's name and checks if the file already exists"""
|
||||
|
||||
title = nameCorrector(POST['postTitle'])
|
||||
FILENAME = title + "_" + POST['postId']
|
||||
PATH = GLOBAL.directory / POST["postSubreddit"]
|
||||
|
||||
possibleExtensions = [".jpg",".png",".mp4",".gif",".webm",".md"]
|
||||
|
||||
for i in range(2):
|
||||
for extension in possibleExtensions:
|
||||
FILE_PATH = PATH / (FILENAME+extension)
|
||||
if FILE_PATH.exists():
|
||||
return True
|
||||
else:
|
||||
FILENAME = POST['postId']
|
||||
for extension in possibleExtensions:
|
||||
|
||||
OLD_FILE_PATH = PATH / (
|
||||
title
|
||||
+ "_" + POST['postId']
|
||||
+ extension
|
||||
)
|
||||
FILE_PATH = PATH / (
|
||||
POST["postSubmitter"]
|
||||
+ "_" + title
|
||||
+ "_" + POST['postId']
|
||||
+ extension
|
||||
)
|
||||
|
||||
SHORT_FILE_PATH = PATH / (POST['postId']+extension)
|
||||
|
||||
if OLD_FILE_PATH.exists() or \
|
||||
FILE_PATH.exists() or \
|
||||
SHORT_FILE_PATH.exists():
|
||||
|
||||
return True
|
||||
|
||||
else:
|
||||
return False
|
||||
|
||||
|
|
@ -523,7 +538,7 @@ def download(submissions):
|
|||
)
|
||||
)
|
||||
|
||||
if postExists(submissions[i]):
|
||||
if isPostExists(submissions[i]):
|
||||
print(submissions[i]['postType'].upper())
|
||||
print("It already exists")
|
||||
duplicates += 1
|
||||
|
|
|
|||
|
|
@ -87,13 +87,14 @@ class Erome:
|
|||
extension = getExtension(IMAGES[0])
|
||||
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(title+"_" +post['postId']+extension)
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+extension)
|
||||
|
||||
fileDir = title + "_" + post['postId'] + extension
|
||||
fileDir = directory / fileDir
|
||||
|
||||
tempDir = title + "_" + post['postId'] + '.tmp'
|
||||
tempDir = directory / tempDir
|
||||
fileDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+extension
|
||||
)
|
||||
tempDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||
)
|
||||
|
||||
imageURL = "https:" + IMAGES[0]
|
||||
|
||||
|
|
@ -106,9 +107,11 @@ class Erome:
|
|||
|
||||
else:
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(title+"_"+post['postId'],end="\n\n")
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
||||
|
||||
folderDir = directory / (title+"_"+post['postId'])
|
||||
folderDir = directory / (
|
||||
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
||||
)
|
||||
|
||||
try:
|
||||
if not os.path.exists(folderDir):
|
||||
|
|
@ -220,13 +223,22 @@ class Imgur:
|
|||
post['postExt'] = getExtension(post['mediaURL'])
|
||||
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(title+"_" +post['postId']+post['postExt'])
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+post['postExt'])
|
||||
|
||||
fileDir = title + "_" + post['postId'] + post['postExt']
|
||||
fileDir = directory / fileDir
|
||||
fileDir = directory / (
|
||||
post["postSubmitter"]
|
||||
+ "_" + title
|
||||
+ "_" + post['postId']
|
||||
+ post['postExt']
|
||||
)
|
||||
|
||||
tempDir = directory / (
|
||||
post["postSubmitter"]
|
||||
+ "_" + title
|
||||
+ "_" + post['postId']
|
||||
+ ".tmp"
|
||||
)
|
||||
|
||||
tempDir = title + "_" + post['postId'] + '.tmp'
|
||||
tempDir = directory / tempDir
|
||||
try:
|
||||
getFile(fileDir,tempDir,post['mediaURL'])
|
||||
except FileNameTooLong:
|
||||
|
|
@ -242,9 +254,11 @@ class Imgur:
|
|||
duplicates = 0
|
||||
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(title+"_"+post['postId'],end="\n\n")
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
||||
|
||||
folderDir = directory / (title+"_"+post['postId'])
|
||||
folderDir = directory / (
|
||||
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
||||
)
|
||||
|
||||
try:
|
||||
if not os.path.exists(folderDir):
|
||||
|
|
@ -369,10 +383,15 @@ class Gfycat:
|
|||
|
||||
if not os.path.exists(directory): os.makedirs(directory)
|
||||
title = nameCorrector(POST['postTitle'])
|
||||
print(title+"_"+POST['postId']+POST['postExt'])
|
||||
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||
|
||||
fileDir = directory / (title+"_"+POST['postId']+POST['postExt'])
|
||||
tempDir = directory / (title+"_"+POST['postId']+".tmp")
|
||||
fileDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||
)
|
||||
tempDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||
)
|
||||
|
||||
try:
|
||||
getFile(fileDir,tempDir,POST['mediaURL'])
|
||||
except FileNameTooLong:
|
||||
|
|
@ -418,13 +437,14 @@ class Direct:
|
|||
POST['postExt'] = getExtension(POST['postURL'])
|
||||
if not os.path.exists(directory): os.makedirs(directory)
|
||||
title = nameCorrector(POST['postTitle'])
|
||||
print(title+"_"+POST['postId']+POST['postExt'])
|
||||
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||
|
||||
fileDir = title+"_"+POST['postId']+POST['postExt']
|
||||
fileDir = directory / fileDir
|
||||
|
||||
tempDir = title+"_"+POST['postId']+".tmp"
|
||||
tempDir = directory / tempDir
|
||||
fileDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||
)
|
||||
tempDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||
)
|
||||
|
||||
try:
|
||||
getFile(fileDir,tempDir,POST['postURL'])
|
||||
|
|
@ -439,10 +459,11 @@ class Self:
|
|||
if not os.path.exists(directory): os.makedirs(directory)
|
||||
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(title+"_"+post['postId']+".md")
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+".md")
|
||||
|
||||
fileDir = title+"_"+post['postId']+".md"
|
||||
fileDir = directory / fileDir
|
||||
fileDir = directory / (
|
||||
post["postSubmitter"]+"_"+title+"_"+post['postId']+".md"
|
||||
)
|
||||
|
||||
if Path.is_file(fileDir):
|
||||
raise FileAlreadyExistsError
|
||||
|
|
@ -465,7 +486,11 @@ class Self:
|
|||
+ ")\n"
|
||||
+ post["postContent"]
|
||||
+ "\n\n---\n\n"
|
||||
+ "submitted by [u/"
|
||||
+ "submitted to [r/"
|
||||
+ post["postSubreddit"]
|
||||
+ "](https://www.reddit.com/r/"
|
||||
+ post["postSubreddit"]
|
||||
+ ") by [u/"
|
||||
+ post["postSubmitter"]
|
||||
+ "](https://www.reddit.com/user/"
|
||||
+ post["postSubmitter"]
|
||||
|
|
|
|||
112
src/searcher.py
112
src/searcher.py
|
|
@ -14,60 +14,62 @@ from src.errors import (NoMatchingSubmissionFound, NoPrawSupport,
|
|||
|
||||
print = printToFile
|
||||
|
||||
class GetAuth:
|
||||
def __init__(self,redditInstance,port):
|
||||
self.redditInstance = redditInstance
|
||||
self.PORT = int(port)
|
||||
|
||||
def recieve_connection(self):
|
||||
"""Wait for and then return a connected socket..
|
||||
Opens a TCP connection on port 8080, and waits for a single client.
|
||||
"""
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server.bind(('localhost', self.PORT))
|
||||
server.listen(1)
|
||||
client = server.accept()[0]
|
||||
server.close()
|
||||
return client
|
||||
|
||||
def send_message(self, message):
|
||||
"""Send message to client and close the connection."""
|
||||
self.client.send('HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8'))
|
||||
self.client.close()
|
||||
|
||||
def getRefreshToken(self,*scopes):
|
||||
state = str(random.randint(0, 65000))
|
||||
url = self.redditInstance.auth.url(scopes, state, 'permanent')
|
||||
print("Go to this URL and login to reddit:\n\n",url)
|
||||
webbrowser.open(url,new=2)
|
||||
|
||||
self.client = self.recieve_connection()
|
||||
data = self.client.recv(1024).decode('utf-8')
|
||||
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
|
||||
params = {
|
||||
key: value for (key, value) in [token.split('=') \
|
||||
for token in param_tokens]
|
||||
}
|
||||
if state != params['state']:
|
||||
self.send_message(
|
||||
client, 'State mismatch. Expected: {} Received: {}'
|
||||
.format(state, params['state'])
|
||||
)
|
||||
raise RedditLoginFailed
|
||||
elif 'error' in params:
|
||||
self.send_message(client, params['error'])
|
||||
raise RedditLoginFailed
|
||||
|
||||
refresh_token = self.redditInstance.auth.authorize(params['code'])
|
||||
self.send_message(
|
||||
"<script>" \
|
||||
"alert(\"You can go back to terminal window now.\");" \
|
||||
"</script>"
|
||||
)
|
||||
return (self.redditInstance,refresh_token)
|
||||
|
||||
def beginPraw(config,user_agent = str(socket.gethostname())):
|
||||
class GetAuth:
|
||||
def __init__(self,redditInstance,port):
|
||||
self.redditInstance = redditInstance
|
||||
self.PORT = int(port)
|
||||
|
||||
def recieve_connection(self):
|
||||
"""Wait for and then return a connected socket..
|
||||
Opens a TCP connection on port 8080, and waits for a single client.
|
||||
"""
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server.bind(('localhost', self.PORT))
|
||||
server.listen(1)
|
||||
client = server.accept()[0]
|
||||
server.close()
|
||||
return client
|
||||
|
||||
def send_message(self, message):
|
||||
"""Send message to client and close the connection."""
|
||||
self.client.send(
|
||||
'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8')
|
||||
)
|
||||
self.client.close()
|
||||
|
||||
def getRefreshToken(self,*scopes):
|
||||
state = str(random.randint(0, 65000))
|
||||
url = self.redditInstance.auth.url(scopes, state, 'permanent')
|
||||
print("Go to this URL and login to reddit:\n\n",url)
|
||||
webbrowser.open(url,new=2)
|
||||
|
||||
self.client = self.recieve_connection()
|
||||
data = self.client.recv(1024).decode('utf-8')
|
||||
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
|
||||
params = {
|
||||
key: value for (key, value) in [token.split('=') \
|
||||
for token in param_tokens]
|
||||
}
|
||||
if state != params['state']:
|
||||
self.send_message(
|
||||
client, 'State mismatch. Expected: {} Received: {}'
|
||||
.format(state, params['state'])
|
||||
)
|
||||
raise RedditLoginFailed
|
||||
elif 'error' in params:
|
||||
self.send_message(client, params['error'])
|
||||
raise RedditLoginFailed
|
||||
|
||||
refresh_token = self.redditInstance.auth.authorize(params['code'])
|
||||
self.send_message(
|
||||
"<script>" \
|
||||
"alert(\"You can go back to terminal window now.\");" \
|
||||
"</script>"
|
||||
)
|
||||
return (self.redditInstance,refresh_token)
|
||||
|
||||
"""Start reddit instance"""
|
||||
|
||||
scopes = ['identity','history','read']
|
||||
|
|
@ -245,8 +247,6 @@ def getPosts(args):
|
|||
raise MultiredditNotFound
|
||||
|
||||
elif "submitted" in args:
|
||||
# TODO
|
||||
# USE REDDIT.USER.ME() INSTEAD WHEN "ME" PASSED AS A --USER
|
||||
print (
|
||||
"submitted posts of {user}\nsort: {sort}\n" \
|
||||
"time: {time}\nlimit: {limit}\n".format(
|
||||
|
|
@ -263,8 +263,6 @@ def getPosts(args):
|
|||
)
|
||||
|
||||
elif "upvoted" in args:
|
||||
# TODO
|
||||
# USE REDDIT.USER.ME() INSTEAD WHEN "ME" PASSED AS A --USER
|
||||
print (
|
||||
"upvoted posts of {user}\nlimit: {limit}\n".format(
|
||||
user=args["user"],
|
||||
|
|
|
|||
Loading…
Reference in a new issue