CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-06-26 10:00:20 +12:00
Code Issues Releases Wiki Activity

Merge pull request #695 from Serene-Arc/bug_fix_677

Browse source 
Closes https://github.com/aliparlakci/bulk-downloader-for-reddit/issues/677
This commit is contained in:
Serene 2022-11-21 14:43:24 +10:00 committed by GitHub
parent 49727aea6e 1385545e26
commit 9ba62f8c97
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 22 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
bdfr/archiver.py
View file

@ -8,6 +8,7 @@ from typing import Iterator
import dict2xml
import praw.models
import prawcore
import yaml
from bdfr.archive_entry.base_archive_entry import BaseArchiveEntry
@ -28,17 +29,20 @@ class Archiver(RedditConnector):
def download(self):
for generator in self.reddit_lists:
for submission in generator:
if (submission.author and submission.author.name in self.args.ignore_user) or \
(submission.author is None and 'DELETED' in self.args.ignore_user):
logger.debug(
f'Submission {submission.id} in {submission.subreddit.display_name} skipped'
f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user')
continue
if submission.id in self.excluded_submission_ids:
logger.debug(f'Object {submission.id} in exclusion list, skipping')
continue
logger.debug(f'Attempting to archive submission {submission.id}')
self.write_entry(submission)
try:
if (submission.author and submission.author.name in self.args.ignore_user) or \
(submission.author is None and 'DELETED' in self.args.ignore_user):
logger.debug(
f'Submission {submission.id} in {submission.subreddit.display_name} skipped'
f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user')
continue
if submission.id in self.excluded_submission_ids:
logger.debug(f'Object {submission.id} in exclusion list, skipping')
continue
logger.debug(f'Attempting to archive submission {submission.id}')
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to be archived due to a PRAW exception: {e}')
def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
supplied_submissions = []

6
bdfr/downloader.py
View file

@ -12,6 +12,7 @@ from pathlib import Path
import praw
import praw.exceptions
import praw.models
import prawcore
from bdfr import exceptions as errors
from bdfr.configuration import Configuration
@ -42,7 +43,10 @@ class RedditDownloader(RedditConnector):
def download(self):
for generator in self.reddit_lists:
for submission in generator:
self._download_submission(submission)
try:
self._download_submission(submission)
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to download due to a PRAW exception: {e}')
def _download_submission(self, submission: praw.models.Submission):
if submission.id in self.excluded_submission_ids:

2
tests/integration_tests/test_download_integration.py
View file

@ -232,6 +232,8 @@ def test_cli_download_long(test_args: list[str], tmp_path: Path):
['--subreddit', 'donaldtrump', '-L', 10], # Banned subreddit
['--user', 'djnish', '--user', 'helen_darten', '-m', 'cuteanimalpics', '-L', 10],
['--subreddit', 'friends', '-L', 10],
['-l', 'ijy4ch'],
['-l', 'kw4wjm'],
))
def test_cli_download_soft_fail(test_args: list[str], tmp_path: Path):
runner = CliRunner()