CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-05-19 19:52:41 +12:00
Code Issues Releases Wiki Activity

Add catch for per-submission praw errors

Browse source
This commit is contained in:
Serene-Arc 2022-11-20 18:54:56 +10:00
parent 49727aea6e
commit 5341d6f12c
2 changed files with 20 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
bdfr/archiver.py
View file

@ -8,6 +8,7 @@ from typing import Iterator
import dict2xml
import praw.models
import prawcore
import yaml
from bdfr.archive_entry.base_archive_entry import BaseArchiveEntry
@ -28,17 +29,20 @@ class Archiver(RedditConnector):
def download(self):
for generator in self.reddit_lists:
for submission in generator:
if (submission.author and submission.author.name in self.args.ignore_user) or \
(submission.author is None and 'DELETED' in self.args.ignore_user):
logger.debug(
f'Submission {submission.id} in {submission.subreddit.display_name} skipped'
f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user')
continue
if submission.id in self.excluded_submission_ids:
logger.debug(f'Object {submission.id} in exclusion list, skipping')
continue
logger.debug(f'Attempting to archive submission {submission.id}')
self.write_entry(submission)
try:
if (submission.author and submission.author.name in self.args.ignore_user) or \
(submission.author is None and 'DELETED' in self.args.ignore_user):
logger.debug(
f'Submission {submission.id} in {submission.subreddit.display_name} skipped'
f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user')
continue
if submission.id in self.excluded_submission_ids:
logger.debug(f'Object {submission.id} in exclusion list, skipping')
continue
logger.debug(f'Attempting to archive submission {submission.id}')
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to be archived due to a PRAW exception: {e}')
def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
supplied_submissions = []

6
bdfr/downloader.py
View file

@ -12,6 +12,7 @@ from pathlib import Path
import praw
import praw.exceptions
import praw.models
import prawcore
from bdfr import exceptions as errors
from bdfr.configuration import Configuration
@ -42,7 +43,10 @@ class RedditDownloader(RedditConnector):
def download(self):
for generator in self.reddit_lists:
for submission in generator:
self._download_submission(submission)
try:
self._download_submission(submission)
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to download due to a PRAW exception: {e}')
def _download_submission(self, submission: praw.models.Submission):
if submission.id in self.excluded_submission_ids: