CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-06-29 11:30:30 +12:00
Code Issues Releases Wiki Activity

Add last few tests for RedditDownloader

Browse source
This commit is contained in:
Serene-Arc 2021-03-10 16:50:19 +10:00 committed by Ali Parlakci
parent d78c4ca78e
commit 1046bcdf69
2 changed files with 69 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
bulkredditdownloader/downloader.py
View file

@ -270,24 +270,20 @@ class RedditDownloader:
logger.error(f'Could not download submission {submission.name}: {e}') logger.error(f'Could not download submission {submission.name}: {e}')
return return
if self.args.no_download: content = downloader.find_resources(self.authenticator)
logger.info(f'Skipping download for submission {submission.id}') for destination, res in self.file_name_formatter.format_resource_paths(content, self.download_directory):
else: if destination.exists():
content = downloader.find_resources(self.authenticator) logger.debug(f'File already exists: {destination}')
for res in content: else:
destination = self.file_name_formatter.format_path(res, self.download_directory) res.download()
if destination.exists(): if res.hash.hexdigest() in self.master_hash_list and self.args.no_dupes:
logger.debug(f'File already exists: {destination}') logger.debug(f'Resource from {res.url} downloaded elsewhere')
else: else:
if res.hash.hexdigest() not in self.master_hash_list and not self.args.no_dupes: # TODO: consider making a hard link/symlink here
# TODO: consider making a hard link/symlink here destination.parent.mkdir(parents=True, exist_ok=True)
destination.parent.mkdir(parents=True, exist_ok=True) with open(destination, 'wb') as file:
with open(destination, 'wb') as file: file.write(res.content)
file.write(res.content) logger.debug(f'Written file to {destination}')
logger.debug(f'Written file to {destination}') self.master_hash_list.append(res.hash.hexdigest())
self.master_hash_list.append(res.hash.hexdigest()) logger.debug(f'Hash added to master list: {res.hash.hexdigest()}')
logger.debug(f'Hash added to master list: {res.hash.hexdigest()}') logger.info(f'Downloaded submission {submission.name}')
else:
logger.debug(f'Resource from {res.url} downloaded elsewhere')
logger.info(f'Downloaded submission {submission.name}')

64
bulkredditdownloader/tests/test_downloader.py
View file

@ -2,6 +2,7 @@
# coding=utf-8 # coding=utf-8
import argparse import argparse
import re
from pathlib import Path from pathlib import Path
from typing import Iterator from typing import Iterator
from unittest.mock import MagicMock from unittest.mock import MagicMock
@ -10,6 +11,7 @@ import praw
import praw.models import praw.models
import pytest import pytest
from bulkredditdownloader.__main__ import _setup_logging
from bulkredditdownloader.download_filter import DownloadFilter from bulkredditdownloader.download_filter import DownloadFilter
from bulkredditdownloader.downloader import RedditDownloader, RedditTypes from bulkredditdownloader.downloader import RedditDownloader, RedditTypes
from bulkredditdownloader.exceptions import BulkDownloaderException, RedditAuthenticationError, RedditUserError from bulkredditdownloader.exceptions import BulkDownloaderException, RedditAuthenticationError, RedditUserError
@ -114,6 +116,8 @@ def test_create_sort_filter(test_sort: str, expected: str, downloader_mock: Magi
@pytest.mark.parametrize(('test_file_scheme', 'test_folder_scheme'), ( @pytest.mark.parametrize(('test_file_scheme', 'test_folder_scheme'), (
('{POSTID}', '{SUBREDDIT}'), ('{POSTID}', '{SUBREDDIT}'),
('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}'), ('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}'),
('{POSTID}', 'test'),
('{POSTID}', ''),
)) ))
def test_create_file_name_formatter(test_file_scheme: str, test_folder_scheme: str, downloader_mock: MagicMock): def test_create_file_name_formatter(test_file_scheme: str, test_folder_scheme: str, downloader_mock: MagicMock):
downloader_mock.args.set_file_scheme = test_file_scheme downloader_mock.args.set_file_scheme = test_file_scheme
@ -127,10 +131,8 @@ def test_create_file_name_formatter(test_file_scheme: str, test_folder_scheme: s
@pytest.mark.parametrize(('test_file_scheme', 'test_folder_scheme'), ( @pytest.mark.parametrize(('test_file_scheme', 'test_folder_scheme'), (
('', ''), ('', ''),
('{POSTID}', ''),
('', '{SUBREDDIT}'), ('', '{SUBREDDIT}'),
('test', '{SUBREDDIT}'), ('test', '{SUBREDDIT}'),
('{POSTID}', 'test'),
)) ))
def test_create_file_name_formatter_bad(test_file_scheme: str, test_folder_scheme: str, downloader_mock: MagicMock): def test_create_file_name_formatter_bad(test_file_scheme: str, test_folder_scheme: str, downloader_mock: MagicMock):
downloader_mock.args.set_file_scheme = test_file_scheme downloader_mock.args.set_file_scheme = test_file_scheme
@ -350,20 +352,60 @@ def test_get_user_saved_unauthenticated(downloader_mock: MagicMock, reddit_insta
@pytest.mark.online @pytest.mark.online
@pytest.mark.reddit @pytest.mark.reddit
@pytest.mark.skip def test_download_submission(downloader_mock: MagicMock, reddit_instance: praw.Reddit, tmp_path: Path):
def test_download_submission(): downloader_mock.reddit_instance = reddit_instance
raise NotImplementedError downloader_mock.download_filter.check_url.return_value = True
downloader_mock.args.set_folder_scheme = ''
downloader_mock.file_name_formatter = RedditDownloader._create_file_name_formatter(downloader_mock)
downloader_mock.download_directory = tmp_path
downloader_mock.master_hash_list = []
submission = downloader_mock.reddit_instance.submission(id='ljyy27')
RedditDownloader._download_submission(downloader_mock, submission)
folder_contents = list(tmp_path.iterdir())
assert len(folder_contents) == 4
@pytest.mark.online @pytest.mark.online
@pytest.mark.reddit @pytest.mark.reddit
@pytest.mark.skip def test_download_submission_file_exists(
def test_download_submission_file_exists(): downloader_mock: MagicMock,
raise NotImplementedError reddit_instance: praw.Reddit,
tmp_path: Path,
capsys: pytest.CaptureFixture):
_setup_logging(3)
downloader_mock.reddit_instance = reddit_instance
downloader_mock.download_filter.check_url.return_value = True
downloader_mock.args.set_folder_scheme = ''
downloader_mock.file_name_formatter = RedditDownloader._create_file_name_formatter(downloader_mock)
downloader_mock.download_directory = tmp_path
downloader_mock.master_hash_list = []
submission = downloader_mock.reddit_instance.submission(id='m1hqw6')
Path(tmp_path, 'Arneeman_Metagaming isn\'t always a bad thing_m1hqw6_1.png').touch()
RedditDownloader._download_submission(downloader_mock, submission)
folder_contents = list(tmp_path.iterdir())
output = capsys.readouterr()
assert len(folder_contents) == 1
assert 'File already exists: ' in output.out
@pytest.mark.online @pytest.mark.online
@pytest.mark.reddit @pytest.mark.reddit
@pytest.mark.skip def test_download_submission_hash_exists(
def test_download_submission_hash_exists(): downloader_mock: MagicMock,
raise NotImplementedError reddit_instance: praw.Reddit,
tmp_path: Path,
capsys: pytest.CaptureFixture):
_setup_logging(3)
downloader_mock.reddit_instance = reddit_instance
downloader_mock.download_filter.check_url.return_value = True
downloader_mock.args.set_folder_scheme = ''
downloader_mock.args.no_dupes = True
downloader_mock.file_name_formatter = RedditDownloader._create_file_name_formatter(downloader_mock)
downloader_mock.download_directory = tmp_path
downloader_mock.master_hash_list = ['a912af8905ae468e0121e9940f797ad7']
submission = downloader_mock.reddit_instance.submission(id='m1hqw6')
RedditDownloader._download_submission(downloader_mock, submission)
folder_contents = list(tmp_path.iterdir())
output = capsys.readouterr()
assert len(folder_contents) == 0
assert re.search(r'Resource from .*? downloaded elsewhere', output.out)