diff --git a/bdfr/__main__.py b/bdfr/__main__.py index 4d78149..29a245c 100644 --- a/bdfr/__main__.py +++ b/bdfr/__main__.py @@ -25,6 +25,7 @@ _common_options = [ click.option('--upvoted', is_flag=True, default=None), click.option('--saved', is_flag=True, default=None), click.option('--search', default=None, type=str), + click.option('--time-format', type=str, default=None), click.option('-u', '--user', type=str, default=None), click.option('-t', '--time', type=click.Choice(('all', 'hour', 'day', 'week', 'month', 'year')), default=None), click.option('-S', '--sort', type=click.Choice(('hot', 'top', 'new', diff --git a/bdfr/configuration.py b/bdfr/configuration.py index c5c7142..9ab9d45 100644 --- a/bdfr/configuration.py +++ b/bdfr/configuration.py @@ -33,6 +33,7 @@ class Configuration(Namespace): self.submitted: bool = False self.subreddit: list[str] = [] self.time: str = 'all' + self.time_format = None self.upvoted: bool = False self.user: Optional[str] = None self.verbose: int = 0 diff --git a/bdfr/default_config.cfg b/bdfr/default_config.cfg index 1bcb02b..b8039a9 100644 --- a/bdfr/default_config.cfg +++ b/bdfr/default_config.cfg @@ -3,4 +3,5 @@ client_id = U-6gk4ZCh3IeNQ client_secret = 7CZHY6AmKweZME5s50SfDGylaPg scopes = identity, history, read, save backup_log_count = 3 -max_wait_time = 120 \ No newline at end of file +max_wait_time = 120 +time_format = ISO \ No newline at end of file diff --git a/bdfr/downloader.py b/bdfr/downloader.py index f0b1977..b20fbf5 100644 --- a/bdfr/downloader.py +++ b/bdfr/downloader.py @@ -105,6 +105,12 @@ class RedditDownloader: logger.log(9, 'Wrote default download wait time download to config file') self.args.max_wait_time = self.cfg_parser.getint('DEFAULT', 'max_wait_time') logger.debug(f'Setting maximum download wait time to {self.args.max_wait_time} seconds') + if self.args.time_format is None: + option = self.cfg_parser.get('DEFAULT', 'time_format', fallback='ISO') + if re.match(r'^[ \'\"]*$', option): + option = 'ISO' + logger.debug(f'Setting datetime format string to {option}') + self.args.time_format = option # Update config on disk with open(self.config_location, 'w') as file: self.cfg_parser.write(file) @@ -358,7 +364,7 @@ class RedditDownloader: raise errors.BulkDownloaderException(f'User {name} is banned') def _create_file_name_formatter(self) -> FileNameFormatter: - return FileNameFormatter(self.args.file_scheme, self.args.folder_scheme) + return FileNameFormatter(self.args.file_scheme, self.args.folder_scheme, self.args.time_format) def _create_time_filter(self) -> RedditTypes.TimeType: try: diff --git a/bdfr/file_name_formatter.py b/bdfr/file_name_formatter.py index e1d42d7..c6c13c2 100644 --- a/bdfr/file_name_formatter.py +++ b/bdfr/file_name_formatter.py @@ -26,18 +26,18 @@ class FileNameFormatter: 'upvotes', ) - def __init__(self, file_format_string: str, directory_format_string: str): + def __init__(self, file_format_string: str, directory_format_string: str, time_format_string: str): if not self.validate_string(file_format_string): raise BulkDownloaderException(f'"{file_format_string}" is not a valid format string') self.file_format_string = file_format_string self.directory_format_string: list[str] = directory_format_string.split('/') + self.time_format_string = time_format_string - @staticmethod - def _format_name(submission: (Comment, Submission), format_string: str) -> str: + def _format_name(self, submission: (Comment, Submission), format_string: str) -> str: if isinstance(submission, Submission): - attributes = FileNameFormatter._generate_name_dict_from_submission(submission) + attributes = self._generate_name_dict_from_submission(submission) elif isinstance(submission, Comment): - attributes = FileNameFormatter._generate_name_dict_from_comment(submission) + attributes = self._generate_name_dict_from_comment(submission) else: raise BulkDownloaderException(f'Cannot name object {type(submission).__name__}') result = format_string @@ -65,8 +65,7 @@ class FileNameFormatter: in_string = in_string.replace(match, converted_match) return in_string - @staticmethod - def _generate_name_dict_from_submission(submission: Submission) -> dict: + def _generate_name_dict_from_submission(self, submission: Submission) -> dict: submission_attributes = { 'title': submission.title, 'subreddit': submission.subreddit.display_name, @@ -74,17 +73,18 @@ class FileNameFormatter: 'postid': submission.id, 'upvotes': submission.score, 'flair': submission.link_flair_text, - 'date': FileNameFormatter._convert_timestamp(submission.created_utc), + 'date': self._convert_timestamp(submission.created_utc), } return submission_attributes - @staticmethod - def _convert_timestamp(timestamp: float) -> str: + def _convert_timestamp(self, timestamp: float) -> str: input_time = datetime.datetime.fromtimestamp(timestamp) - return input_time.isoformat() + if self.time_format_string.upper().strip() == 'ISO': + return input_time.isoformat() + else: + return input_time.strftime(self.time_format_string) - @staticmethod - def _generate_name_dict_from_comment(comment: Comment) -> dict: + def _generate_name_dict_from_comment(self, comment: Comment) -> dict: comment_attributes = { 'title': comment.submission.title, 'subreddit': comment.subreddit.display_name, @@ -92,7 +92,7 @@ class FileNameFormatter: 'postid': comment.id, 'upvotes': comment.score, 'flair': '', - 'date': FileNameFormatter._convert_timestamp(comment.created_utc), + 'date': self._convert_timestamp(comment.created_utc), } return comment_attributes @@ -160,9 +160,8 @@ class FileNameFormatter: result = any([f'{{{key}}}' in test_string.lower() for key in FileNameFormatter.key_terms]) if result: if 'POSTID' not in test_string: - logger.warning( - 'Some files might not be downloaded due to name conflicts as filenames are' - ' not guaranteed to be be unique without {POSTID}') + logger.warning('Some files might not be downloaded due to name conflicts as filenames are' + ' not guaranteed to be be unique without {POSTID}') return True else: return False diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 0a3418e..f1a20fc 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -22,6 +22,7 @@ from bdfr.site_authenticator import SiteAuthenticator @pytest.fixture() def args() -> Configuration: args = Configuration() + args.time_format = 'ISO' return args diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index bcb38d7..b4035dd 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -32,7 +32,7 @@ def reddit_submission(reddit_instance: praw.Reddit) -> praw.models.Submission: return reddit_instance.submission(id='lgilgt') -@pytest.mark.parametrize(('format_string', 'expected'), ( +@pytest.mark.parametrize(('test_format_string', 'expected'), ( ('{SUBREDDIT}', 'randomreddit'), ('{REDDITOR}', 'person'), ('{POSTID}', '12345'), @@ -40,10 +40,10 @@ def reddit_submission(reddit_instance: praw.Reddit) -> praw.models.Submission: ('{FLAIR}', 'test_flair'), ('{DATE}', '2021-04-21T09:30:00'), ('{REDDITOR}_{TITLE}_{POSTID}', 'person_name_12345'), - ('{RANDOM}', '{RANDOM}'), )) -def test_format_name_mock(format_string: str, expected: str, submission: MagicMock): - result = FileNameFormatter._format_name(submission, format_string) +def test_format_name_mock(test_format_string: str, expected: str, submission: MagicMock): + test_formatter = FileNameFormatter(test_format_string, '', 'ISO') + result = test_formatter._format_name(submission, test_format_string) assert result == expected @@ -63,7 +63,7 @@ def test_check_format_string_validity(test_string: str, expected: bool): @pytest.mark.online @pytest.mark.reddit -@pytest.mark.parametrize(('format_string', 'expected'), ( +@pytest.mark.parametrize(('test_format_string', 'expected'), ( ('{SUBREDDIT}', 'Mindustry'), ('{REDDITOR}', 'Gamer_player_boi'), ('{POSTID}', 'lgilgt'), @@ -71,8 +71,9 @@ def test_check_format_string_validity(test_string: str, expected: bool): ('{SUBREDDIT}_{TITLE}', 'Mindustry_Toxopid that is NOT humane >:('), ('{REDDITOR}_{TITLE}_{POSTID}', 'Gamer_player_boi_Toxopid that is NOT humane >:(_lgilgt') )) -def test_format_name_real(format_string: str, expected: str, reddit_submission: praw.models.Submission): - result = FileNameFormatter._format_name(reddit_submission, format_string) +def test_format_name_real(test_format_string: str, expected: str, reddit_submission: praw.models.Submission): + test_formatter = FileNameFormatter(test_format_string, '', '') + result = test_formatter._format_name(reddit_submission, test_format_string) assert result == expected @@ -101,7 +102,7 @@ def test_format_full( expected: str, reddit_submission: praw.models.Submission): test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png') - test_formatter = FileNameFormatter(format_string_file, format_string_directory) + test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO') result = test_formatter.format_path(test_resource, Path('test')) assert str(result) == expected @@ -118,7 +119,7 @@ def test_format_full_conform( format_string_file: str, reddit_submission: praw.models.Submission): test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png') - test_formatter = FileNameFormatter(format_string_file, format_string_directory) + test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO') test_formatter.format_path(test_resource, Path('test')) @@ -138,7 +139,7 @@ def test_format_full_with_index_suffix( reddit_submission: praw.models.Submission, ): test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png') - test_formatter = FileNameFormatter(format_string_file, format_string_directory) + test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO') result = test_formatter.format_path(test_resource, Path('test'), index) assert str(result) == expected @@ -152,7 +153,7 @@ def test_format_multiple_resources(): new_mock.source_submission.title = 'test' new_mock.source_submission.__class__ = praw.models.Submission mocks.append(new_mock) - test_formatter = FileNameFormatter('{TITLE}', '') + test_formatter = FileNameFormatter('{TITLE}', '', 'ISO') results = test_formatter.format_resource_paths(mocks, Path('.')) results = set([str(res[0]) for res in results]) assert results == {'test_1.png', 'test_2.png', 'test_3.png', 'test_4.png'} @@ -196,7 +197,7 @@ def test_shorten_filenames(submission: MagicMock, tmp_path: Path): submission.subreddit.display_name = 'test' submission.id = 'BBBBBB' test_resource = Resource(submission, 'www.example.com/empty', '.jpeg') - test_formatter = FileNameFormatter('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}') + test_formatter = FileNameFormatter('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}', 'ISO') result = test_formatter.format_path(test_resource, tmp_path) result.parent.mkdir(parents=True) result.touch() @@ -237,7 +238,8 @@ def test_strip_emojies(test_string: str, expected: str): )) def test_generate_dict_for_submission(test_submission_id: str, expected: dict, reddit_instance: praw.Reddit): test_submission = reddit_instance.submission(id=test_submission_id) - result = FileNameFormatter._generate_name_dict_from_submission(test_submission) + test_formatter = FileNameFormatter('{TITLE}', '', 'ISO') + result = test_formatter._generate_name_dict_from_submission(test_submission) assert all([result.get(key) == expected[key] for key in expected.keys()]) @@ -253,7 +255,8 @@ def test_generate_dict_for_submission(test_submission_id: str, expected: dict, r )) def test_generate_dict_for_comment(test_comment_id: str, expected: dict, reddit_instance: praw.Reddit): test_comment = reddit_instance.comment(id=test_comment_id) - result = FileNameFormatter._generate_name_dict_from_comment(test_comment) + test_formatter = FileNameFormatter('{TITLE}', '', 'ISO') + result = test_formatter._generate_name_dict_from_comment(test_comment) assert all([result.get(key) == expected[key] for key in expected.keys()]) @@ -272,7 +275,7 @@ def test_format_archive_entry_comment( reddit_instance: praw.Reddit, ): test_comment = reddit_instance.comment(id=test_comment_id) - test_formatter = FileNameFormatter(test_file_scheme, test_folder_scheme) + test_formatter = FileNameFormatter(test_file_scheme, test_folder_scheme, 'ISO') test_entry = Resource(test_comment, '', '.json') result = test_formatter.format_path(test_entry, tmp_path) assert result.name == expected_name @@ -288,7 +291,7 @@ def test_multilevel_folder_scheme( tmp_path: Path, submission: MagicMock, ): - test_formatter = FileNameFormatter('{POSTID}', test_folder_scheme) + test_formatter = FileNameFormatter('{POSTID}', test_folder_scheme, 'ISO') test_resource = MagicMock() test_resource.source_submission = submission test_resource.extension = '.png' @@ -308,7 +311,8 @@ def test_multilevel_folder_scheme( )) def test_preserve_emojis(test_name_string: str, expected: str, submission: MagicMock): submission.title = test_name_string - result = FileNameFormatter._format_name(submission, '{TITLE}') + test_formatter = FileNameFormatter('{TITLE}', '', 'ISO') + result = test_formatter._format_name(submission, '{TITLE}') assert result == expected @@ -328,5 +332,18 @@ def test_convert_unicode_escapes(test_string: str, expected: str): )) def test_convert_timestamp(test_datetime: datetime, expected: str): test_timestamp = test_datetime.timestamp() - result = FileNameFormatter._convert_timestamp(test_timestamp) + test_formatter = FileNameFormatter('{POSTID}', '', 'ISO') + result = test_formatter._convert_timestamp(test_timestamp) + assert result == expected + + +@pytest.mark.parametrize(('test_time_format', 'expected'), ( + ('ISO', '2021-05-02T13:33:00'), + ('%Y_%m', '2021_05'), + ('%Y-%m-%d', '2021-05-02'), +)) +def test_time_string_formats(test_time_format: str, expected: str): + test_time = datetime(2021, 5, 2, 13, 33) + test_formatter = FileNameFormatter('{TITLE}', '', test_time_format) + result = test_formatter._convert_timestamp(test_time.timestamp()) assert result == expected