CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-06-21 11:40:21 +12:00
Code Issues Releases Wiki Activity

Add customisable time formatting

Browse source
This commit is contained in:
Serene-Arc 2021-05-02 13:56:39 +10:00 committed by Serene
parent eda12e5274
commit afa3e2548f
7 changed files with 63 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
bdfr/__main__.py
View file

@ -25,6 +25,7 @@ _common_options = [
click.option('--upvoted', is_flag=True, default=None),
click.option('--saved', is_flag=True, default=None),
click.option('--search', default=None, type=str),
click.option('--time-format', type=str, default=None),
click.option('-u', '--user', type=str, default=None),
click.option('-t', '--time', type=click.Choice(('all', 'hour', 'day', 'week', 'month', 'year')), default=None),
click.option('-S', '--sort', type=click.Choice(('hot', 'top', 'new',

1
bdfr/configuration.py
View file

@ -33,6 +33,7 @@ class Configuration(Namespace):
self.submitted: bool = False
self.subreddit: list[str] = []
self.time: str = 'all'
self.time_format = None
self.upvoted: bool = False
self.user: Optional[str] = None
self.verbose: int = 0

3
bdfr/default_config.cfg
View file

@ -3,4 +3,5 @@ client_id = U-6gk4ZCh3IeNQ
client_secret = 7CZHY6AmKweZME5s50SfDGylaPg
scopes = identity, history, read, save
backup_log_count = 3
max_wait_time = 120
max_wait_time = 120
time_format = ISO

8
bdfr/downloader.py
View file

@ -105,6 +105,12 @@ class RedditDownloader:
logger.log(9, 'Wrote default download wait time download to config file')
self.args.max_wait_time = self.cfg_parser.getint('DEFAULT', 'max_wait_time')
logger.debug(f'Setting maximum download wait time to {self.args.max_wait_time} seconds')
if self.args.time_format is None:
option = self.cfg_parser.get('DEFAULT', 'time_format', fallback='ISO')
if re.match(r'^[ \'\"]*$', option):
option = 'ISO'
logger.debug(f'Setting datetime format string to {option}')
self.args.time_format = option
# Update config on disk
with open(self.config_location, 'w') as file:
self.cfg_parser.write(file)
@ -358,7 +364,7 @@ class RedditDownloader:
raise errors.BulkDownloaderException(f'User {name} is banned')
def _create_file_name_formatter(self) -> FileNameFormatter:
return FileNameFormatter(self.args.file_scheme, self.args.folder_scheme)
return FileNameFormatter(self.args.file_scheme, self.args.folder_scheme, self.args.time_format)
def _create_time_filter(self) -> RedditTypes.TimeType:
try:

33
bdfr/file_name_formatter.py
View file

@ -26,18 +26,18 @@ class FileNameFormatter:
'upvotes',
)
def __init__(self, file_format_string: str, directory_format_string: str):
def __init__(self, file_format_string: str, directory_format_string: str, time_format_string: str):
if not self.validate_string(file_format_string):
raise BulkDownloaderException(f'"{file_format_string}" is not a valid format string')
self.file_format_string = file_format_string
self.directory_format_string: list[str] = directory_format_string.split('/')
self.time_format_string = time_format_string
@staticmethod
def _format_name(submission: (Comment, Submission), format_string: str) -> str:
def _format_name(self, submission: (Comment, Submission), format_string: str) -> str:
if isinstance(submission, Submission):
attributes = FileNameFormatter._generate_name_dict_from_submission(submission)
attributes = self._generate_name_dict_from_submission(submission)
elif isinstance(submission, Comment):
attributes = FileNameFormatter._generate_name_dict_from_comment(submission)
attributes = self._generate_name_dict_from_comment(submission)
else:
raise BulkDownloaderException(f'Cannot name object {type(submission).__name__}')
result = format_string
@ -65,8 +65,7 @@ class FileNameFormatter:
in_string = in_string.replace(match, converted_match)
return in_string
@staticmethod
def _generate_name_dict_from_submission(submission: Submission) -> dict:
def _generate_name_dict_from_submission(self, submission: Submission) -> dict:
submission_attributes = {
'title': submission.title,
'subreddit': submission.subreddit.display_name,
@ -74,17 +73,18 @@ class FileNameFormatter:
'postid': submission.id,
'upvotes': submission.score,
'flair': submission.link_flair_text,
'date': FileNameFormatter._convert_timestamp(submission.created_utc),
'date': self._convert_timestamp(submission.created_utc),
}
return submission_attributes
@staticmethod
def _convert_timestamp(timestamp: float) -> str:
def _convert_timestamp(self, timestamp: float) -> str:
input_time = datetime.datetime.fromtimestamp(timestamp)
return input_time.isoformat()
if self.time_format_string.upper().strip() == 'ISO':
return input_time.isoformat()
else:
return input_time.strftime(self.time_format_string)
@staticmethod
def _generate_name_dict_from_comment(comment: Comment) -> dict:
def _generate_name_dict_from_comment(self, comment: Comment) -> dict:
comment_attributes = {
'title': comment.submission.title,
'subreddit': comment.subreddit.display_name,
@ -92,7 +92,7 @@ class FileNameFormatter:
'postid': comment.id,
'upvotes': comment.score,
'flair': '',
'date': FileNameFormatter._convert_timestamp(comment.created_utc),
'date': self._convert_timestamp(comment.created_utc),
}
return comment_attributes
@ -160,9 +160,8 @@ class FileNameFormatter:
result = any([f'{{{key}}}' in test_string.lower() for key in FileNameFormatter.key_terms])
if result:
if 'POSTID' not in test_string:
logger.warning(
'Some files might not be downloaded due to name conflicts as filenames are'
' not guaranteed to be be unique without {POSTID}')
logger.warning('Some files might not be downloaded due to name conflicts as filenames are'
' not guaranteed to be be unique without {POSTID}')
return True
else:
return False

1
tests/test_downloader.py
View file

@ -22,6 +22,7 @@ from bdfr.site_authenticator import SiteAuthenticator
@pytest.fixture()
def args() -> Configuration:
args = Configuration()
args.time_format = 'ISO'
return args

53
tests/test_file_name_formatter.py
View file

@ -32,7 +32,7 @@ def reddit_submission(reddit_instance: praw.Reddit) -> praw.models.Submission:
return reddit_instance.submission(id='lgilgt')
@pytest.mark.parametrize(('format_string', 'expected'), (
@pytest.mark.parametrize(('test_format_string', 'expected'), (
('{SUBREDDIT}', 'randomreddit'),
('{REDDITOR}', 'person'),
('{POSTID}', '12345'),
@ -40,10 +40,10 @@ def reddit_submission(reddit_instance: praw.Reddit) -> praw.models.Submission:
('{FLAIR}', 'test_flair'),
('{DATE}', '2021-04-21T09:30:00'),
('{REDDITOR}_{TITLE}_{POSTID}', 'person_name_12345'),
('{RANDOM}', '{RANDOM}'),
))
def test_format_name_mock(format_string: str, expected: str, submission: MagicMock):
result = FileNameFormatter._format_name(submission, format_string)
def test_format_name_mock(test_format_string: str, expected: str, submission: MagicMock):
test_formatter = FileNameFormatter(test_format_string, '', 'ISO')
result = test_formatter._format_name(submission, test_format_string)
assert result == expected
@ -63,7 +63,7 @@ def test_check_format_string_validity(test_string: str, expected: bool):
@pytest.mark.online
@pytest.mark.reddit
@pytest.mark.parametrize(('format_string', 'expected'), (
@pytest.mark.parametrize(('test_format_string', 'expected'), (
('{SUBREDDIT}', 'Mindustry'),
('{REDDITOR}', 'Gamer_player_boi'),
('{POSTID}', 'lgilgt'),
@ -71,8 +71,9 @@ def test_check_format_string_validity(test_string: str, expected: bool):
('{SUBREDDIT}_{TITLE}', 'Mindustry_Toxopid that is NOT humane >:('),
('{REDDITOR}_{TITLE}_{POSTID}', 'Gamer_player_boi_Toxopid that is NOT humane >:(_lgilgt')
))
def test_format_name_real(format_string: str, expected: str, reddit_submission: praw.models.Submission):
result = FileNameFormatter._format_name(reddit_submission, format_string)
def test_format_name_real(test_format_string: str, expected: str, reddit_submission: praw.models.Submission):
test_formatter = FileNameFormatter(test_format_string, '', '')
result = test_formatter._format_name(reddit_submission, test_format_string)
assert result == expected
@ -101,7 +102,7 @@ def test_format_full(
expected: str,
reddit_submission: praw.models.Submission):
test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
test_formatter = FileNameFormatter(format_string_file, format_string_directory)
test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO')
result = test_formatter.format_path(test_resource, Path('test'))
assert str(result) == expected
@ -118,7 +119,7 @@ def test_format_full_conform(
format_string_file: str,
reddit_submission: praw.models.Submission):
test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
test_formatter = FileNameFormatter(format_string_file, format_string_directory)
test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO')
test_formatter.format_path(test_resource, Path('test'))
@ -138,7 +139,7 @@ def test_format_full_with_index_suffix(
reddit_submission: praw.models.Submission,
):
test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
test_formatter = FileNameFormatter(format_string_file, format_string_directory)
test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO')
result = test_formatter.format_path(test_resource, Path('test'), index)
assert str(result) == expected
@ -152,7 +153,7 @@ def test_format_multiple_resources():
new_mock.source_submission.title = 'test'
new_mock.source_submission.__class__ = praw.models.Submission
mocks.append(new_mock)
test_formatter = FileNameFormatter('{TITLE}', '')
test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
results = test_formatter.format_resource_paths(mocks, Path('.'))
results = set([str(res[0]) for res in results])
assert results == {'test_1.png', 'test_2.png', 'test_3.png', 'test_4.png'}
@ -196,7 +197,7 @@ def test_shorten_filenames(submission: MagicMock, tmp_path: Path):
submission.subreddit.display_name = 'test'
submission.id = 'BBBBBB'
test_resource = Resource(submission, 'www.example.com/empty', '.jpeg')
test_formatter = FileNameFormatter('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}')
test_formatter = FileNameFormatter('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}', 'ISO')
result = test_formatter.format_path(test_resource, tmp_path)
result.parent.mkdir(parents=True)
result.touch()
@ -237,7 +238,8 @@ def test_strip_emojies(test_string: str, expected: str):
))
def test_generate_dict_for_submission(test_submission_id: str, expected: dict, reddit_instance: praw.Reddit):
test_submission = reddit_instance.submission(id=test_submission_id)
result = FileNameFormatter._generate_name_dict_from_submission(test_submission)
test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
result = test_formatter._generate_name_dict_from_submission(test_submission)
assert all([result.get(key) == expected[key] for key in expected.keys()])
@ -253,7 +255,8 @@ def test_generate_dict_for_submission(test_submission_id: str, expected: dict, r
))
def test_generate_dict_for_comment(test_comment_id: str, expected: dict, reddit_instance: praw.Reddit):
test_comment = reddit_instance.comment(id=test_comment_id)
result = FileNameFormatter._generate_name_dict_from_comment(test_comment)
test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
result = test_formatter._generate_name_dict_from_comment(test_comment)
assert all([result.get(key) == expected[key] for key in expected.keys()])
@ -272,7 +275,7 @@ def test_format_archive_entry_comment(
reddit_instance: praw.Reddit,
):
test_comment = reddit_instance.comment(id=test_comment_id)
test_formatter = FileNameFormatter(test_file_scheme, test_folder_scheme)
test_formatter = FileNameFormatter(test_file_scheme, test_folder_scheme, 'ISO')
test_entry = Resource(test_comment, '', '.json')
result = test_formatter.format_path(test_entry, tmp_path)
assert result.name == expected_name
@ -288,7 +291,7 @@ def test_multilevel_folder_scheme(
tmp_path: Path,
submission: MagicMock,
):
test_formatter = FileNameFormatter('{POSTID}', test_folder_scheme)
test_formatter = FileNameFormatter('{POSTID}', test_folder_scheme, 'ISO')
test_resource = MagicMock()
test_resource.source_submission = submission
test_resource.extension = '.png'
@ -308,7 +311,8 @@ def test_multilevel_folder_scheme(
))
def test_preserve_emojis(test_name_string: str, expected: str, submission: MagicMock):
submission.title = test_name_string
result = FileNameFormatter._format_name(submission, '{TITLE}')
test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
result = test_formatter._format_name(submission, '{TITLE}')
assert result == expected
@ -328,5 +332,18 @@ def test_convert_unicode_escapes(test_string: str, expected: str):
))
def test_convert_timestamp(test_datetime: datetime, expected: str):
test_timestamp = test_datetime.timestamp()
result = FileNameFormatter._convert_timestamp(test_timestamp)
test_formatter = FileNameFormatter('{POSTID}', '', 'ISO')
result = test_formatter._convert_timestamp(test_timestamp)
assert result == expected
@pytest.mark.parametrize(('test_time_format', 'expected'), (
('ISO', '2021-05-02T13:33:00'),
('%Y_%m', '2021_05'),
('%Y-%m-%d', '2021-05-02'),
))
def test_time_string_formats(test_time_format: str, expected: str):
test_time = datetime(2021, 5, 2, 13, 33)
test_formatter = FileNameFormatter('{TITLE}', '', test_time_format)
result = test_formatter._convert_timestamp(test_time.timestamp())
assert result == expected