CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-05-29 00:19:59 +12:00
Code Issues Releases Wiki Activity

Add 5xx soft fail for clone/archive

Browse source
This commit is contained in:
OMEGARAZER 2022-12-10 21:19:29 -05:00
parent 3aa740e979
commit ac91c9089c
No known key found for this signature in database
GPG key ID: D89925310D306E35
2 changed files with 35 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
bdfr/archiver.py
View file

@ -4,6 +4,7 @@
import json
import logging
import re
from time import sleep
from typing import Iterator, Union
import dict2xml
@ -28,23 +29,28 @@ class Archiver(RedditConnector):
def download(self):
for generator in self.reddit_lists:
for submission in generator:
try:
if (submission.author and submission.author.name in self.args.ignore_user) or (
submission.author is None and "DELETED" in self.args.ignore_user
):
logger.debug(
f"Submission {submission.id} in {submission.subreddit.display_name} skipped"
f" due to {submission.author.name if submission.author else 'DELETED'} being an ignored user"
)
continue
if submission.id in self.excluded_submission_ids:
logger.debug(f"Object {submission.id} in exclusion list, skipping")
continue
logger.debug(f"Attempting to archive submission {submission.id}")
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}")
try:
for submission in generator:
try:
if (submission.author and submission.author.name in self.args.ignore_user) or (
submission.author is None and "DELETED" in self.args.ignore_user
):
logger.debug(
f"Submission {submission.id} in {submission.subreddit.display_name} skipped due to"
f" {submission.author.name if submission.author else 'DELETED'} being an ignored user"
)
continue
if submission.id in self.excluded_submission_ids:
logger.debug(f"Object {submission.id} in exclusion list, skipping")
continue
logger.debug(f"Attempting to archive submission {submission.id}")
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
supplied_submissions = []

18
bdfr/cloner.py
View file

@ -2,6 +2,7 @@
# coding=utf-8
import logging
from time import sleep
import prawcore
@ -18,9 +19,14 @@ class RedditCloner(RedditDownloader, Archiver):
def download(self):
for generator in self.reddit_lists:
for submission in generator:
try:
self._download_submission(submission)
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
try:
for submission in generator:
try:
self._download_submission(submission)
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)