Add last few tests for RedditDownloader
This commit is contained in:
parent
d78c4ca78e
commit
1046bcdf69
|
@ -270,16 +270,15 @@ class RedditDownloader:
|
||||||
logger.error(f'Could not download submission {submission.name}: {e}')
|
logger.error(f'Could not download submission {submission.name}: {e}')
|
||||||
return
|
return
|
||||||
|
|
||||||
if self.args.no_download:
|
|
||||||
logger.info(f'Skipping download for submission {submission.id}')
|
|
||||||
else:
|
|
||||||
content = downloader.find_resources(self.authenticator)
|
content = downloader.find_resources(self.authenticator)
|
||||||
for res in content:
|
for destination, res in self.file_name_formatter.format_resource_paths(content, self.download_directory):
|
||||||
destination = self.file_name_formatter.format_path(res, self.download_directory)
|
|
||||||
if destination.exists():
|
if destination.exists():
|
||||||
logger.debug(f'File already exists: {destination}')
|
logger.debug(f'File already exists: {destination}')
|
||||||
else:
|
else:
|
||||||
if res.hash.hexdigest() not in self.master_hash_list and not self.args.no_dupes:
|
res.download()
|
||||||
|
if res.hash.hexdigest() in self.master_hash_list and self.args.no_dupes:
|
||||||
|
logger.debug(f'Resource from {res.url} downloaded elsewhere')
|
||||||
|
else:
|
||||||
# TODO: consider making a hard link/symlink here
|
# TODO: consider making a hard link/symlink here
|
||||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
with open(destination, 'wb') as file:
|
with open(destination, 'wb') as file:
|
||||||
|
@ -287,7 +286,4 @@ class RedditDownloader:
|
||||||
logger.debug(f'Written file to {destination}')
|
logger.debug(f'Written file to {destination}')
|
||||||
self.master_hash_list.append(res.hash.hexdigest())
|
self.master_hash_list.append(res.hash.hexdigest())
|
||||||
logger.debug(f'Hash added to master list: {res.hash.hexdigest()}')
|
logger.debug(f'Hash added to master list: {res.hash.hexdigest()}')
|
||||||
else:
|
|
||||||
logger.debug(f'Resource from {res.url} downloaded elsewhere')
|
|
||||||
|
|
||||||
logger.info(f'Downloaded submission {submission.name}')
|
logger.info(f'Downloaded submission {submission.name}')
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Iterator
|
from typing import Iterator
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
@ -10,6 +11,7 @@ import praw
|
||||||
import praw.models
|
import praw.models
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from bulkredditdownloader.__main__ import _setup_logging
|
||||||
from bulkredditdownloader.download_filter import DownloadFilter
|
from bulkredditdownloader.download_filter import DownloadFilter
|
||||||
from bulkredditdownloader.downloader import RedditDownloader, RedditTypes
|
from bulkredditdownloader.downloader import RedditDownloader, RedditTypes
|
||||||
from bulkredditdownloader.exceptions import BulkDownloaderException, RedditAuthenticationError, RedditUserError
|
from bulkredditdownloader.exceptions import BulkDownloaderException, RedditAuthenticationError, RedditUserError
|
||||||
|
@ -114,6 +116,8 @@ def test_create_sort_filter(test_sort: str, expected: str, downloader_mock: Magi
|
||||||
@pytest.mark.parametrize(('test_file_scheme', 'test_folder_scheme'), (
|
@pytest.mark.parametrize(('test_file_scheme', 'test_folder_scheme'), (
|
||||||
('{POSTID}', '{SUBREDDIT}'),
|
('{POSTID}', '{SUBREDDIT}'),
|
||||||
('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}'),
|
('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}'),
|
||||||
|
('{POSTID}', 'test'),
|
||||||
|
('{POSTID}', ''),
|
||||||
))
|
))
|
||||||
def test_create_file_name_formatter(test_file_scheme: str, test_folder_scheme: str, downloader_mock: MagicMock):
|
def test_create_file_name_formatter(test_file_scheme: str, test_folder_scheme: str, downloader_mock: MagicMock):
|
||||||
downloader_mock.args.set_file_scheme = test_file_scheme
|
downloader_mock.args.set_file_scheme = test_file_scheme
|
||||||
|
@ -127,10 +131,8 @@ def test_create_file_name_formatter(test_file_scheme: str, test_folder_scheme: s
|
||||||
|
|
||||||
@pytest.mark.parametrize(('test_file_scheme', 'test_folder_scheme'), (
|
@pytest.mark.parametrize(('test_file_scheme', 'test_folder_scheme'), (
|
||||||
('', ''),
|
('', ''),
|
||||||
('{POSTID}', ''),
|
|
||||||
('', '{SUBREDDIT}'),
|
('', '{SUBREDDIT}'),
|
||||||
('test', '{SUBREDDIT}'),
|
('test', '{SUBREDDIT}'),
|
||||||
('{POSTID}', 'test'),
|
|
||||||
))
|
))
|
||||||
def test_create_file_name_formatter_bad(test_file_scheme: str, test_folder_scheme: str, downloader_mock: MagicMock):
|
def test_create_file_name_formatter_bad(test_file_scheme: str, test_folder_scheme: str, downloader_mock: MagicMock):
|
||||||
downloader_mock.args.set_file_scheme = test_file_scheme
|
downloader_mock.args.set_file_scheme = test_file_scheme
|
||||||
|
@ -350,20 +352,60 @@ def test_get_user_saved_unauthenticated(downloader_mock: MagicMock, reddit_insta
|
||||||
|
|
||||||
@pytest.mark.online
|
@pytest.mark.online
|
||||||
@pytest.mark.reddit
|
@pytest.mark.reddit
|
||||||
@pytest.mark.skip
|
def test_download_submission(downloader_mock: MagicMock, reddit_instance: praw.Reddit, tmp_path: Path):
|
||||||
def test_download_submission():
|
downloader_mock.reddit_instance = reddit_instance
|
||||||
raise NotImplementedError
|
downloader_mock.download_filter.check_url.return_value = True
|
||||||
|
downloader_mock.args.set_folder_scheme = ''
|
||||||
|
downloader_mock.file_name_formatter = RedditDownloader._create_file_name_formatter(downloader_mock)
|
||||||
|
downloader_mock.download_directory = tmp_path
|
||||||
|
downloader_mock.master_hash_list = []
|
||||||
|
submission = downloader_mock.reddit_instance.submission(id='ljyy27')
|
||||||
|
RedditDownloader._download_submission(downloader_mock, submission)
|
||||||
|
folder_contents = list(tmp_path.iterdir())
|
||||||
|
assert len(folder_contents) == 4
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.online
|
@pytest.mark.online
|
||||||
@pytest.mark.reddit
|
@pytest.mark.reddit
|
||||||
@pytest.mark.skip
|
def test_download_submission_file_exists(
|
||||||
def test_download_submission_file_exists():
|
downloader_mock: MagicMock,
|
||||||
raise NotImplementedError
|
reddit_instance: praw.Reddit,
|
||||||
|
tmp_path: Path,
|
||||||
|
capsys: pytest.CaptureFixture):
|
||||||
|
_setup_logging(3)
|
||||||
|
downloader_mock.reddit_instance = reddit_instance
|
||||||
|
downloader_mock.download_filter.check_url.return_value = True
|
||||||
|
downloader_mock.args.set_folder_scheme = ''
|
||||||
|
downloader_mock.file_name_formatter = RedditDownloader._create_file_name_formatter(downloader_mock)
|
||||||
|
downloader_mock.download_directory = tmp_path
|
||||||
|
downloader_mock.master_hash_list = []
|
||||||
|
submission = downloader_mock.reddit_instance.submission(id='m1hqw6')
|
||||||
|
Path(tmp_path, 'Arneeman_Metagaming isn\'t always a bad thing_m1hqw6_1.png').touch()
|
||||||
|
RedditDownloader._download_submission(downloader_mock, submission)
|
||||||
|
folder_contents = list(tmp_path.iterdir())
|
||||||
|
output = capsys.readouterr()
|
||||||
|
assert len(folder_contents) == 1
|
||||||
|
assert 'File already exists: ' in output.out
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.online
|
@pytest.mark.online
|
||||||
@pytest.mark.reddit
|
@pytest.mark.reddit
|
||||||
@pytest.mark.skip
|
def test_download_submission_hash_exists(
|
||||||
def test_download_submission_hash_exists():
|
downloader_mock: MagicMock,
|
||||||
raise NotImplementedError
|
reddit_instance: praw.Reddit,
|
||||||
|
tmp_path: Path,
|
||||||
|
capsys: pytest.CaptureFixture):
|
||||||
|
_setup_logging(3)
|
||||||
|
downloader_mock.reddit_instance = reddit_instance
|
||||||
|
downloader_mock.download_filter.check_url.return_value = True
|
||||||
|
downloader_mock.args.set_folder_scheme = ''
|
||||||
|
downloader_mock.args.no_dupes = True
|
||||||
|
downloader_mock.file_name_formatter = RedditDownloader._create_file_name_formatter(downloader_mock)
|
||||||
|
downloader_mock.download_directory = tmp_path
|
||||||
|
downloader_mock.master_hash_list = ['a912af8905ae468e0121e9940f797ad7']
|
||||||
|
submission = downloader_mock.reddit_instance.submission(id='m1hqw6')
|
||||||
|
RedditDownloader._download_submission(downloader_mock, submission)
|
||||||
|
folder_contents = list(tmp_path.iterdir())
|
||||||
|
output = capsys.readouterr()
|
||||||
|
assert len(folder_contents) == 0
|
||||||
|
assert re.search(r'Resource from .*? downloaded elsewhere', output.out)
|
||||||
|
|
Loading…
Reference in a new issue