2021-06-06 22:29:09 +12:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
import logging
|
2023-01-26 16:23:59 +13:00
|
|
|
from collections.abc import Iterable
|
2022-12-11 15:19:29 +13:00
|
|
|
from time import sleep
|
2021-06-06 22:29:09 +12:00
|
|
|
|
2022-11-24 13:48:17 +13:00
|
|
|
import prawcore
|
|
|
|
|
2021-06-06 22:29:09 +12:00
|
|
|
from bdfr.archiver import Archiver
|
|
|
|
from bdfr.configuration import Configuration
|
|
|
|
from bdfr.downloader import RedditDownloader
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
class RedditCloner(RedditDownloader, Archiver):
|
2023-01-08 16:33:35 +13:00
|
|
|
def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handler] = ()):
|
2023-01-29 17:58:36 +13:00
|
|
|
super().__init__(args, logging_handlers)
|
2021-06-06 22:29:09 +12:00
|
|
|
|
|
|
|
def download(self):
|
|
|
|
for generator in self.reddit_lists:
|
2022-12-11 15:19:29 +13:00
|
|
|
try:
|
|
|
|
for submission in generator:
|
|
|
|
try:
|
|
|
|
self._download_submission(submission)
|
|
|
|
self.write_entry(submission)
|
|
|
|
except prawcore.PrawcoreException as e:
|
|
|
|
logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
|
|
|
|
except prawcore.PrawcoreException as e:
|
|
|
|
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
|
|
|
|
logger.debug("Waiting 60 seconds to continue")
|
|
|
|
sleep(60)
|