Add customisable time formatting

bdfr/__main__.py

@@ -25,6 +25,7 @@ _common_options = [
     click.option('--upvoted', is_flag=True, default=None),
     click.option('--saved', is_flag=True, default=None),
     click.option('--search', default=None, type=str),
+    click.option('--time-format', type=str, default=None),
     click.option('-u', '--user', type=str, default=None),
     click.option('-t', '--time', type=click.Choice(('all', 'hour', 'day', 'week', 'month', 'year')), default=None),
     click.option('-S', '--sort', type=click.Choice(('hot', 'top', 'new',

bdfr/configuration.py

@@ -33,6 +33,7 @@ class Configuration(Namespace):
         self.submitted: bool = False
         self.subreddit: list[str] = []
         self.time: str = 'all'
+        self.time_format = None
         self.upvoted: bool = False
         self.user: Optional[str] = None
         self.verbose: int = 0

bdfr/default_config.cfg

@@ -3,4 +3,5 @@ client_id = U-6gk4ZCh3IeNQ
 client_secret = 7CZHY6AmKweZME5s50SfDGylaPg
 scopes = identity, history, read, save
 backup_log_count = 3
-max_wait_time = 120
+max_wait_time = 120
+time_format = ISO

bdfr/downloader.py

@@ -105,6 +105,12 @@ class RedditDownloader:
                 logger.log(9, 'Wrote default download wait time download to config file')
             self.args.max_wait_time = self.cfg_parser.getint('DEFAULT', 'max_wait_time')
             logger.debug(f'Setting maximum download wait time to {self.args.max_wait_time} seconds')
+        if self.args.time_format is None:
+            option = self.cfg_parser.get('DEFAULT', 'time_format', fallback='ISO')
+            if re.match(r'^[ \'\"]*$', option):
+                option = 'ISO'
+            logger.debug(f'Setting datetime format string to {option}')
+            self.args.time_format = option
         # Update config on disk
         with open(self.config_location, 'w') as file:
             self.cfg_parser.write(file)
@@ -358,7 +364,7 @@ class RedditDownloader:
                 raise errors.BulkDownloaderException(f'User {name} is banned')
 
     def _create_file_name_formatter(self) -> FileNameFormatter:
-        return FileNameFormatter(self.args.file_scheme, self.args.folder_scheme)
+        return FileNameFormatter(self.args.file_scheme, self.args.folder_scheme, self.args.time_format)
 
     def _create_time_filter(self) -> RedditTypes.TimeType:
         try:

bdfr/file_name_formatter.py

@@ -26,18 +26,18 @@ class FileNameFormatter:
         'upvotes',
     )
 
-    def __init__(self, file_format_string: str, directory_format_string: str):
+    def __init__(self, file_format_string: str, directory_format_string: str, time_format_string: str):
         if not self.validate_string(file_format_string):
             raise BulkDownloaderException(f'"{file_format_string}" is not a valid format string')
         self.file_format_string = file_format_string
         self.directory_format_string: list[str] = directory_format_string.split('/')
+        self.time_format_string = time_format_string
 
-    @staticmethod
+    def _format_name(self, submission: (Comment, Submission), format_string: str) -> str:
-    def _format_name(submission: (Comment, Submission), format_string: str) -> str:
         if isinstance(submission, Submission):
-            attributes = FileNameFormatter._generate_name_dict_from_submission(submission)
+            attributes = self._generate_name_dict_from_submission(submission)
         elif isinstance(submission, Comment):
-            attributes = FileNameFormatter._generate_name_dict_from_comment(submission)
+            attributes = self._generate_name_dict_from_comment(submission)
         else:
             raise BulkDownloaderException(f'Cannot name object {type(submission).__name__}')
         result = format_string
@@ -65,8 +65,7 @@ class FileNameFormatter:
                 in_string = in_string.replace(match, converted_match)
         return in_string
 
-    @staticmethod
+    def _generate_name_dict_from_submission(self, submission: Submission) -> dict:
-    def _generate_name_dict_from_submission(submission: Submission) -> dict:
         submission_attributes = {
             'title': submission.title,
             'subreddit': submission.subreddit.display_name,
@@ -74,17 +73,18 @@ class FileNameFormatter:
             'postid': submission.id,
             'upvotes': submission.score,
             'flair': submission.link_flair_text,
-            'date': FileNameFormatter._convert_timestamp(submission.created_utc),
+            'date': self._convert_timestamp(submission.created_utc),
         }
         return submission_attributes
 
-    @staticmethod
+    def _convert_timestamp(self, timestamp: float) -> str:
-    def _convert_timestamp(timestamp: float) -> str:
         input_time = datetime.datetime.fromtimestamp(timestamp)
-        return input_time.isoformat()
+        if self.time_format_string.upper().strip() == 'ISO':
+            return input_time.isoformat()
+        else:
+            return input_time.strftime(self.time_format_string)
 
-    @staticmethod
+    def _generate_name_dict_from_comment(self, comment: Comment) -> dict:
-    def _generate_name_dict_from_comment(comment: Comment) -> dict:
         comment_attributes = {
             'title': comment.submission.title,
             'subreddit': comment.subreddit.display_name,
@@ -92,7 +92,7 @@ class FileNameFormatter:
             'postid': comment.id,
             'upvotes': comment.score,
             'flair': '',
-            'date': FileNameFormatter._convert_timestamp(comment.created_utc),
+            'date': self._convert_timestamp(comment.created_utc),
         }
         return comment_attributes
 
@@ -160,9 +160,8 @@ class FileNameFormatter:
         result = any([f'{{{key}}}' in test_string.lower() for key in FileNameFormatter.key_terms])
         if result:
             if 'POSTID' not in test_string:
-                logger.warning(
+                logger.warning('Some files might not be downloaded due to name conflicts as filenames are'
-                    'Some files might not be downloaded due to name conflicts as filenames are'
+                               ' not guaranteed to be be unique without {POSTID}')
-                    ' not guaranteed to be be unique without {POSTID}')
             return True
         else:
             return False

tests/test_downloader.py

@@ -22,6 +22,7 @@ from bdfr.site_authenticator import SiteAuthenticator
 @pytest.fixture()
 def args() -> Configuration:
     args = Configuration()
+    args.time_format = 'ISO'
     return args
 
 

tests/test_file_name_formatter.py

@@ -32,7 +32,7 @@ def reddit_submission(reddit_instance: praw.Reddit) -> praw.models.Submission:
     return reddit_instance.submission(id='lgilgt')
 
 
-@pytest.mark.parametrize(('format_string', 'expected'), (
+@pytest.mark.parametrize(('test_format_string', 'expected'), (
     ('{SUBREDDIT}', 'randomreddit'),
     ('{REDDITOR}', 'person'),
     ('{POSTID}', '12345'),
@@ -40,10 +40,10 @@ def reddit_submission(reddit_instance: praw.Reddit) -> praw.models.Submission:
     ('{FLAIR}', 'test_flair'),
     ('{DATE}', '2021-04-21T09:30:00'),
     ('{REDDITOR}_{TITLE}_{POSTID}', 'person_name_12345'),
-    ('{RANDOM}', '{RANDOM}'),
 ))
-def test_format_name_mock(format_string: str, expected: str, submission: MagicMock):
+def test_format_name_mock(test_format_string: str, expected: str, submission: MagicMock):
-    result = FileNameFormatter._format_name(submission, format_string)
+    test_formatter = FileNameFormatter(test_format_string, '', 'ISO')
+    result = test_formatter._format_name(submission, test_format_string)
     assert result == expected
 
 
@@ -63,7 +63,7 @@ def test_check_format_string_validity(test_string: str, expected: bool):
 
 @pytest.mark.online
 @pytest.mark.reddit
-@pytest.mark.parametrize(('format_string', 'expected'), (
+@pytest.mark.parametrize(('test_format_string', 'expected'), (
     ('{SUBREDDIT}', 'Mindustry'),
     ('{REDDITOR}', 'Gamer_player_boi'),
     ('{POSTID}', 'lgilgt'),
@@ -71,8 +71,9 @@ def test_check_format_string_validity(test_string: str, expected: bool):
     ('{SUBREDDIT}_{TITLE}', 'Mindustry_Toxopid that is NOT humane >:('),
     ('{REDDITOR}_{TITLE}_{POSTID}', 'Gamer_player_boi_Toxopid that is NOT humane >:(_lgilgt')
 ))
-def test_format_name_real(format_string: str, expected: str, reddit_submission: praw.models.Submission):
+def test_format_name_real(test_format_string: str, expected: str, reddit_submission: praw.models.Submission):
-    result = FileNameFormatter._format_name(reddit_submission, format_string)
+    test_formatter = FileNameFormatter(test_format_string, '', '')
+    result = test_formatter._format_name(reddit_submission, test_format_string)
     assert result == expected
 
 
@@ -101,7 +102,7 @@ def test_format_full(
         expected: str,
         reddit_submission: praw.models.Submission):
     test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
-    test_formatter = FileNameFormatter(format_string_file, format_string_directory)
+    test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO')
     result = test_formatter.format_path(test_resource, Path('test'))
     assert str(result) == expected
 
@@ -118,7 +119,7 @@ def test_format_full_conform(
         format_string_file: str,
         reddit_submission: praw.models.Submission):
     test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
-    test_formatter = FileNameFormatter(format_string_file, format_string_directory)
+    test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO')
     test_formatter.format_path(test_resource, Path('test'))
 
 
@@ -138,7 +139,7 @@ def test_format_full_with_index_suffix(
         reddit_submission: praw.models.Submission,
 ):
     test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
-    test_formatter = FileNameFormatter(format_string_file, format_string_directory)
+    test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO')
     result = test_formatter.format_path(test_resource, Path('test'), index)
     assert str(result) == expected
 
@@ -152,7 +153,7 @@ def test_format_multiple_resources():
         new_mock.source_submission.title = 'test'
         new_mock.source_submission.__class__ = praw.models.Submission
         mocks.append(new_mock)
-    test_formatter = FileNameFormatter('{TITLE}', '')
+    test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
     results = test_formatter.format_resource_paths(mocks, Path('.'))
     results = set([str(res[0]) for res in results])
     assert results == {'test_1.png', 'test_2.png', 'test_3.png', 'test_4.png'}
@@ -196,7 +197,7 @@ def test_shorten_filenames(submission: MagicMock, tmp_path: Path):
     submission.subreddit.display_name = 'test'
     submission.id = 'BBBBBB'
     test_resource = Resource(submission, 'www.example.com/empty', '.jpeg')
-    test_formatter = FileNameFormatter('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}')
+    test_formatter = FileNameFormatter('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}', 'ISO')
     result = test_formatter.format_path(test_resource, tmp_path)
     result.parent.mkdir(parents=True)
     result.touch()
@@ -237,7 +238,8 @@ def test_strip_emojies(test_string: str, expected: str):
 ))
 def test_generate_dict_for_submission(test_submission_id: str, expected: dict, reddit_instance: praw.Reddit):
     test_submission = reddit_instance.submission(id=test_submission_id)
-    result = FileNameFormatter._generate_name_dict_from_submission(test_submission)
+    test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
+    result = test_formatter._generate_name_dict_from_submission(test_submission)
     assert all([result.get(key) == expected[key] for key in expected.keys()])
 
 
@@ -253,7 +255,8 @@ def test_generate_dict_for_submission(test_submission_id: str, expected: dict, r
 ))
 def test_generate_dict_for_comment(test_comment_id: str, expected: dict, reddit_instance: praw.Reddit):
     test_comment = reddit_instance.comment(id=test_comment_id)
-    result = FileNameFormatter._generate_name_dict_from_comment(test_comment)
+    test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
+    result = test_formatter._generate_name_dict_from_comment(test_comment)
     assert all([result.get(key) == expected[key] for key in expected.keys()])
 
 
@@ -272,7 +275,7 @@ def test_format_archive_entry_comment(
         reddit_instance: praw.Reddit,
 ):
     test_comment = reddit_instance.comment(id=test_comment_id)
-    test_formatter = FileNameFormatter(test_file_scheme, test_folder_scheme)
+    test_formatter = FileNameFormatter(test_file_scheme, test_folder_scheme, 'ISO')
     test_entry = Resource(test_comment, '', '.json')
     result = test_formatter.format_path(test_entry, tmp_path)
     assert result.name == expected_name
@@ -288,7 +291,7 @@ def test_multilevel_folder_scheme(
         tmp_path: Path,
         submission: MagicMock,
 ):
-    test_formatter = FileNameFormatter('{POSTID}', test_folder_scheme)
+    test_formatter = FileNameFormatter('{POSTID}', test_folder_scheme, 'ISO')
     test_resource = MagicMock()
     test_resource.source_submission = submission
     test_resource.extension = '.png'
@@ -308,7 +311,8 @@ def test_multilevel_folder_scheme(
 ))
 def test_preserve_emojis(test_name_string: str, expected: str, submission: MagicMock):
     submission.title = test_name_string
-    result = FileNameFormatter._format_name(submission, '{TITLE}')
+    test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
+    result = test_formatter._format_name(submission, '{TITLE}')
     assert result == expected
 
 
@@ -328,5 +332,18 @@ def test_convert_unicode_escapes(test_string: str, expected: str):
 ))
 def test_convert_timestamp(test_datetime: datetime, expected: str):
     test_timestamp = test_datetime.timestamp()
-    result = FileNameFormatter._convert_timestamp(test_timestamp)
+    test_formatter = FileNameFormatter('{POSTID}', '', 'ISO')
+    result = test_formatter._convert_timestamp(test_timestamp)
+    assert result == expected
+
+
+@pytest.mark.parametrize(('test_time_format', 'expected'), (
+    ('ISO', '2021-05-02T13:33:00'),
+    ('%Y_%m', '2021_05'),
+    ('%Y-%m-%d', '2021-05-02'),
+))
+def test_time_string_formats(test_time_format: str, expected: str):
+    test_time = datetime(2021, 5, 2, 13, 33)
+    test_formatter = FileNameFormatter('{TITLE}', '', test_time_format)
+    result = test_formatter._convert_timestamp(test_time.timestamp())
     assert result == expected