From 5341d6f12caa4cc8deebff1cb849ac5b0fa0c18e Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Sun, 20 Nov 2022 18:54:56 +1000 Subject: [PATCH 1/2] Add catch for per-submission praw errors --- bdfr/archiver.py | 26 +++++++++++++++----------- bdfr/downloader.py | 6 +++++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/bdfr/archiver.py b/bdfr/archiver.py index 214111f..4bd24f5 100644 --- a/bdfr/archiver.py +++ b/bdfr/archiver.py @@ -8,6 +8,7 @@ from typing import Iterator import dict2xml import praw.models +import prawcore import yaml from bdfr.archive_entry.base_archive_entry import BaseArchiveEntry @@ -28,17 +29,20 @@ class Archiver(RedditConnector): def download(self): for generator in self.reddit_lists: for submission in generator: - if (submission.author and submission.author.name in self.args.ignore_user) or \ - (submission.author is None and 'DELETED' in self.args.ignore_user): - logger.debug( - f'Submission {submission.id} in {submission.subreddit.display_name} skipped' - f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user') - continue - if submission.id in self.excluded_submission_ids: - logger.debug(f'Object {submission.id} in exclusion list, skipping') - continue - logger.debug(f'Attempting to archive submission {submission.id}') - self.write_entry(submission) + try: + if (submission.author and submission.author.name in self.args.ignore_user) or \ + (submission.author is None and 'DELETED' in self.args.ignore_user): + logger.debug( + f'Submission {submission.id} in {submission.subreddit.display_name} skipped' + f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user') + continue + if submission.id in self.excluded_submission_ids: + logger.debug(f'Object {submission.id} in exclusion list, skipping') + continue + logger.debug(f'Attempting to archive submission {submission.id}') + self.write_entry(submission) + except prawcore.PrawcoreException as e: + logger.error(f'Submission {submission.id} failed to be archived due to a PRAW exception: {e}') def get_submissions_from_link(self) -> list[list[praw.models.Submission]]: supplied_submissions = [] diff --git a/bdfr/downloader.py b/bdfr/downloader.py index 7709add..057f6af 100644 --- a/bdfr/downloader.py +++ b/bdfr/downloader.py @@ -12,6 +12,7 @@ from pathlib import Path import praw import praw.exceptions import praw.models +import prawcore from bdfr import exceptions as errors from bdfr.configuration import Configuration @@ -42,7 +43,10 @@ class RedditDownloader(RedditConnector): def download(self): for generator in self.reddit_lists: for submission in generator: - self._download_submission(submission) + try: + self._download_submission(submission) + except prawcore.PrawcoreException as e: + logger.error(f'Submission {submission.id} failed to download due to a PRAW exception: {e}') def _download_submission(self, submission: praw.models.Submission): if submission.id in self.excluded_submission_ids: From 1385545e26a745b9fae422a5bd75af499d131e9b Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Mon, 21 Nov 2022 14:35:57 +1000 Subject: [PATCH 2/2] Add tests for downloader --- tests/integration_tests/test_download_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py index a9f0e0e..44f62c2 100644 --- a/tests/integration_tests/test_download_integration.py +++ b/tests/integration_tests/test_download_integration.py @@ -232,6 +232,8 @@ def test_cli_download_long(test_args: list[str], tmp_path: Path): ['--subreddit', 'donaldtrump', '-L', 10], # Banned subreddit ['--user', 'djnish', '--user', 'helen_darten', '-m', 'cuteanimalpics', '-L', 10], ['--subreddit', 'friends', '-L', 10], + ['-l', 'ijy4ch'], + ['-l', 'kw4wjm'], )) def test_cli_download_soft_fail(test_args: list[str], tmp_path: Path): runner = CliRunner()