Add customisable time formatting
This commit is contained in:
parent
eda12e5274
commit
afa3e2548f
7 changed files with 63 additions and 37 deletions
|
@ -25,6 +25,7 @@ _common_options = [
|
||||||
click.option('--upvoted', is_flag=True, default=None),
|
click.option('--upvoted', is_flag=True, default=None),
|
||||||
click.option('--saved', is_flag=True, default=None),
|
click.option('--saved', is_flag=True, default=None),
|
||||||
click.option('--search', default=None, type=str),
|
click.option('--search', default=None, type=str),
|
||||||
|
click.option('--time-format', type=str, default=None),
|
||||||
click.option('-u', '--user', type=str, default=None),
|
click.option('-u', '--user', type=str, default=None),
|
||||||
click.option('-t', '--time', type=click.Choice(('all', 'hour', 'day', 'week', 'month', 'year')), default=None),
|
click.option('-t', '--time', type=click.Choice(('all', 'hour', 'day', 'week', 'month', 'year')), default=None),
|
||||||
click.option('-S', '--sort', type=click.Choice(('hot', 'top', 'new',
|
click.option('-S', '--sort', type=click.Choice(('hot', 'top', 'new',
|
||||||
|
|
|
@ -33,6 +33,7 @@ class Configuration(Namespace):
|
||||||
self.submitted: bool = False
|
self.submitted: bool = False
|
||||||
self.subreddit: list[str] = []
|
self.subreddit: list[str] = []
|
||||||
self.time: str = 'all'
|
self.time: str = 'all'
|
||||||
|
self.time_format = None
|
||||||
self.upvoted: bool = False
|
self.upvoted: bool = False
|
||||||
self.user: Optional[str] = None
|
self.user: Optional[str] = None
|
||||||
self.verbose: int = 0
|
self.verbose: int = 0
|
||||||
|
|
|
@ -4,3 +4,4 @@ client_secret = 7CZHY6AmKweZME5s50SfDGylaPg
|
||||||
scopes = identity, history, read, save
|
scopes = identity, history, read, save
|
||||||
backup_log_count = 3
|
backup_log_count = 3
|
||||||
max_wait_time = 120
|
max_wait_time = 120
|
||||||
|
time_format = ISO
|
|
@ -105,6 +105,12 @@ class RedditDownloader:
|
||||||
logger.log(9, 'Wrote default download wait time download to config file')
|
logger.log(9, 'Wrote default download wait time download to config file')
|
||||||
self.args.max_wait_time = self.cfg_parser.getint('DEFAULT', 'max_wait_time')
|
self.args.max_wait_time = self.cfg_parser.getint('DEFAULT', 'max_wait_time')
|
||||||
logger.debug(f'Setting maximum download wait time to {self.args.max_wait_time} seconds')
|
logger.debug(f'Setting maximum download wait time to {self.args.max_wait_time} seconds')
|
||||||
|
if self.args.time_format is None:
|
||||||
|
option = self.cfg_parser.get('DEFAULT', 'time_format', fallback='ISO')
|
||||||
|
if re.match(r'^[ \'\"]*$', option):
|
||||||
|
option = 'ISO'
|
||||||
|
logger.debug(f'Setting datetime format string to {option}')
|
||||||
|
self.args.time_format = option
|
||||||
# Update config on disk
|
# Update config on disk
|
||||||
with open(self.config_location, 'w') as file:
|
with open(self.config_location, 'w') as file:
|
||||||
self.cfg_parser.write(file)
|
self.cfg_parser.write(file)
|
||||||
|
@ -358,7 +364,7 @@ class RedditDownloader:
|
||||||
raise errors.BulkDownloaderException(f'User {name} is banned')
|
raise errors.BulkDownloaderException(f'User {name} is banned')
|
||||||
|
|
||||||
def _create_file_name_formatter(self) -> FileNameFormatter:
|
def _create_file_name_formatter(self) -> FileNameFormatter:
|
||||||
return FileNameFormatter(self.args.file_scheme, self.args.folder_scheme)
|
return FileNameFormatter(self.args.file_scheme, self.args.folder_scheme, self.args.time_format)
|
||||||
|
|
||||||
def _create_time_filter(self) -> RedditTypes.TimeType:
|
def _create_time_filter(self) -> RedditTypes.TimeType:
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -26,18 +26,18 @@ class FileNameFormatter:
|
||||||
'upvotes',
|
'upvotes',
|
||||||
)
|
)
|
||||||
|
|
||||||
def __init__(self, file_format_string: str, directory_format_string: str):
|
def __init__(self, file_format_string: str, directory_format_string: str, time_format_string: str):
|
||||||
if not self.validate_string(file_format_string):
|
if not self.validate_string(file_format_string):
|
||||||
raise BulkDownloaderException(f'"{file_format_string}" is not a valid format string')
|
raise BulkDownloaderException(f'"{file_format_string}" is not a valid format string')
|
||||||
self.file_format_string = file_format_string
|
self.file_format_string = file_format_string
|
||||||
self.directory_format_string: list[str] = directory_format_string.split('/')
|
self.directory_format_string: list[str] = directory_format_string.split('/')
|
||||||
|
self.time_format_string = time_format_string
|
||||||
|
|
||||||
@staticmethod
|
def _format_name(self, submission: (Comment, Submission), format_string: str) -> str:
|
||||||
def _format_name(submission: (Comment, Submission), format_string: str) -> str:
|
|
||||||
if isinstance(submission, Submission):
|
if isinstance(submission, Submission):
|
||||||
attributes = FileNameFormatter._generate_name_dict_from_submission(submission)
|
attributes = self._generate_name_dict_from_submission(submission)
|
||||||
elif isinstance(submission, Comment):
|
elif isinstance(submission, Comment):
|
||||||
attributes = FileNameFormatter._generate_name_dict_from_comment(submission)
|
attributes = self._generate_name_dict_from_comment(submission)
|
||||||
else:
|
else:
|
||||||
raise BulkDownloaderException(f'Cannot name object {type(submission).__name__}')
|
raise BulkDownloaderException(f'Cannot name object {type(submission).__name__}')
|
||||||
result = format_string
|
result = format_string
|
||||||
|
@ -65,8 +65,7 @@ class FileNameFormatter:
|
||||||
in_string = in_string.replace(match, converted_match)
|
in_string = in_string.replace(match, converted_match)
|
||||||
return in_string
|
return in_string
|
||||||
|
|
||||||
@staticmethod
|
def _generate_name_dict_from_submission(self, submission: Submission) -> dict:
|
||||||
def _generate_name_dict_from_submission(submission: Submission) -> dict:
|
|
||||||
submission_attributes = {
|
submission_attributes = {
|
||||||
'title': submission.title,
|
'title': submission.title,
|
||||||
'subreddit': submission.subreddit.display_name,
|
'subreddit': submission.subreddit.display_name,
|
||||||
|
@ -74,17 +73,18 @@ class FileNameFormatter:
|
||||||
'postid': submission.id,
|
'postid': submission.id,
|
||||||
'upvotes': submission.score,
|
'upvotes': submission.score,
|
||||||
'flair': submission.link_flair_text,
|
'flair': submission.link_flair_text,
|
||||||
'date': FileNameFormatter._convert_timestamp(submission.created_utc),
|
'date': self._convert_timestamp(submission.created_utc),
|
||||||
}
|
}
|
||||||
return submission_attributes
|
return submission_attributes
|
||||||
|
|
||||||
@staticmethod
|
def _convert_timestamp(self, timestamp: float) -> str:
|
||||||
def _convert_timestamp(timestamp: float) -> str:
|
|
||||||
input_time = datetime.datetime.fromtimestamp(timestamp)
|
input_time = datetime.datetime.fromtimestamp(timestamp)
|
||||||
return input_time.isoformat()
|
if self.time_format_string.upper().strip() == 'ISO':
|
||||||
|
return input_time.isoformat()
|
||||||
|
else:
|
||||||
|
return input_time.strftime(self.time_format_string)
|
||||||
|
|
||||||
@staticmethod
|
def _generate_name_dict_from_comment(self, comment: Comment) -> dict:
|
||||||
def _generate_name_dict_from_comment(comment: Comment) -> dict:
|
|
||||||
comment_attributes = {
|
comment_attributes = {
|
||||||
'title': comment.submission.title,
|
'title': comment.submission.title,
|
||||||
'subreddit': comment.subreddit.display_name,
|
'subreddit': comment.subreddit.display_name,
|
||||||
|
@ -92,7 +92,7 @@ class FileNameFormatter:
|
||||||
'postid': comment.id,
|
'postid': comment.id,
|
||||||
'upvotes': comment.score,
|
'upvotes': comment.score,
|
||||||
'flair': '',
|
'flair': '',
|
||||||
'date': FileNameFormatter._convert_timestamp(comment.created_utc),
|
'date': self._convert_timestamp(comment.created_utc),
|
||||||
}
|
}
|
||||||
return comment_attributes
|
return comment_attributes
|
||||||
|
|
||||||
|
@ -160,9 +160,8 @@ class FileNameFormatter:
|
||||||
result = any([f'{{{key}}}' in test_string.lower() for key in FileNameFormatter.key_terms])
|
result = any([f'{{{key}}}' in test_string.lower() for key in FileNameFormatter.key_terms])
|
||||||
if result:
|
if result:
|
||||||
if 'POSTID' not in test_string:
|
if 'POSTID' not in test_string:
|
||||||
logger.warning(
|
logger.warning('Some files might not be downloaded due to name conflicts as filenames are'
|
||||||
'Some files might not be downloaded due to name conflicts as filenames are'
|
' not guaranteed to be be unique without {POSTID}')
|
||||||
' not guaranteed to be be unique without {POSTID}')
|
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
return False
|
return False
|
||||||
|
|
|
@ -22,6 +22,7 @@ from bdfr.site_authenticator import SiteAuthenticator
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
def args() -> Configuration:
|
def args() -> Configuration:
|
||||||
args = Configuration()
|
args = Configuration()
|
||||||
|
args.time_format = 'ISO'
|
||||||
return args
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -32,7 +32,7 @@ def reddit_submission(reddit_instance: praw.Reddit) -> praw.models.Submission:
|
||||||
return reddit_instance.submission(id='lgilgt')
|
return reddit_instance.submission(id='lgilgt')
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(('format_string', 'expected'), (
|
@pytest.mark.parametrize(('test_format_string', 'expected'), (
|
||||||
('{SUBREDDIT}', 'randomreddit'),
|
('{SUBREDDIT}', 'randomreddit'),
|
||||||
('{REDDITOR}', 'person'),
|
('{REDDITOR}', 'person'),
|
||||||
('{POSTID}', '12345'),
|
('{POSTID}', '12345'),
|
||||||
|
@ -40,10 +40,10 @@ def reddit_submission(reddit_instance: praw.Reddit) -> praw.models.Submission:
|
||||||
('{FLAIR}', 'test_flair'),
|
('{FLAIR}', 'test_flair'),
|
||||||
('{DATE}', '2021-04-21T09:30:00'),
|
('{DATE}', '2021-04-21T09:30:00'),
|
||||||
('{REDDITOR}_{TITLE}_{POSTID}', 'person_name_12345'),
|
('{REDDITOR}_{TITLE}_{POSTID}', 'person_name_12345'),
|
||||||
('{RANDOM}', '{RANDOM}'),
|
|
||||||
))
|
))
|
||||||
def test_format_name_mock(format_string: str, expected: str, submission: MagicMock):
|
def test_format_name_mock(test_format_string: str, expected: str, submission: MagicMock):
|
||||||
result = FileNameFormatter._format_name(submission, format_string)
|
test_formatter = FileNameFormatter(test_format_string, '', 'ISO')
|
||||||
|
result = test_formatter._format_name(submission, test_format_string)
|
||||||
assert result == expected
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
@ -63,7 +63,7 @@ def test_check_format_string_validity(test_string: str, expected: bool):
|
||||||
|
|
||||||
@pytest.mark.online
|
@pytest.mark.online
|
||||||
@pytest.mark.reddit
|
@pytest.mark.reddit
|
||||||
@pytest.mark.parametrize(('format_string', 'expected'), (
|
@pytest.mark.parametrize(('test_format_string', 'expected'), (
|
||||||
('{SUBREDDIT}', 'Mindustry'),
|
('{SUBREDDIT}', 'Mindustry'),
|
||||||
('{REDDITOR}', 'Gamer_player_boi'),
|
('{REDDITOR}', 'Gamer_player_boi'),
|
||||||
('{POSTID}', 'lgilgt'),
|
('{POSTID}', 'lgilgt'),
|
||||||
|
@ -71,8 +71,9 @@ def test_check_format_string_validity(test_string: str, expected: bool):
|
||||||
('{SUBREDDIT}_{TITLE}', 'Mindustry_Toxopid that is NOT humane >:('),
|
('{SUBREDDIT}_{TITLE}', 'Mindustry_Toxopid that is NOT humane >:('),
|
||||||
('{REDDITOR}_{TITLE}_{POSTID}', 'Gamer_player_boi_Toxopid that is NOT humane >:(_lgilgt')
|
('{REDDITOR}_{TITLE}_{POSTID}', 'Gamer_player_boi_Toxopid that is NOT humane >:(_lgilgt')
|
||||||
))
|
))
|
||||||
def test_format_name_real(format_string: str, expected: str, reddit_submission: praw.models.Submission):
|
def test_format_name_real(test_format_string: str, expected: str, reddit_submission: praw.models.Submission):
|
||||||
result = FileNameFormatter._format_name(reddit_submission, format_string)
|
test_formatter = FileNameFormatter(test_format_string, '', '')
|
||||||
|
result = test_formatter._format_name(reddit_submission, test_format_string)
|
||||||
assert result == expected
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
@ -101,7 +102,7 @@ def test_format_full(
|
||||||
expected: str,
|
expected: str,
|
||||||
reddit_submission: praw.models.Submission):
|
reddit_submission: praw.models.Submission):
|
||||||
test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
|
test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
|
||||||
test_formatter = FileNameFormatter(format_string_file, format_string_directory)
|
test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO')
|
||||||
result = test_formatter.format_path(test_resource, Path('test'))
|
result = test_formatter.format_path(test_resource, Path('test'))
|
||||||
assert str(result) == expected
|
assert str(result) == expected
|
||||||
|
|
||||||
|
@ -118,7 +119,7 @@ def test_format_full_conform(
|
||||||
format_string_file: str,
|
format_string_file: str,
|
||||||
reddit_submission: praw.models.Submission):
|
reddit_submission: praw.models.Submission):
|
||||||
test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
|
test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
|
||||||
test_formatter = FileNameFormatter(format_string_file, format_string_directory)
|
test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO')
|
||||||
test_formatter.format_path(test_resource, Path('test'))
|
test_formatter.format_path(test_resource, Path('test'))
|
||||||
|
|
||||||
|
|
||||||
|
@ -138,7 +139,7 @@ def test_format_full_with_index_suffix(
|
||||||
reddit_submission: praw.models.Submission,
|
reddit_submission: praw.models.Submission,
|
||||||
):
|
):
|
||||||
test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
|
test_resource = Resource(reddit_submission, 'i.reddit.com/blabla.png')
|
||||||
test_formatter = FileNameFormatter(format_string_file, format_string_directory)
|
test_formatter = FileNameFormatter(format_string_file, format_string_directory, 'ISO')
|
||||||
result = test_formatter.format_path(test_resource, Path('test'), index)
|
result = test_formatter.format_path(test_resource, Path('test'), index)
|
||||||
assert str(result) == expected
|
assert str(result) == expected
|
||||||
|
|
||||||
|
@ -152,7 +153,7 @@ def test_format_multiple_resources():
|
||||||
new_mock.source_submission.title = 'test'
|
new_mock.source_submission.title = 'test'
|
||||||
new_mock.source_submission.__class__ = praw.models.Submission
|
new_mock.source_submission.__class__ = praw.models.Submission
|
||||||
mocks.append(new_mock)
|
mocks.append(new_mock)
|
||||||
test_formatter = FileNameFormatter('{TITLE}', '')
|
test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
|
||||||
results = test_formatter.format_resource_paths(mocks, Path('.'))
|
results = test_formatter.format_resource_paths(mocks, Path('.'))
|
||||||
results = set([str(res[0]) for res in results])
|
results = set([str(res[0]) for res in results])
|
||||||
assert results == {'test_1.png', 'test_2.png', 'test_3.png', 'test_4.png'}
|
assert results == {'test_1.png', 'test_2.png', 'test_3.png', 'test_4.png'}
|
||||||
|
@ -196,7 +197,7 @@ def test_shorten_filenames(submission: MagicMock, tmp_path: Path):
|
||||||
submission.subreddit.display_name = 'test'
|
submission.subreddit.display_name = 'test'
|
||||||
submission.id = 'BBBBBB'
|
submission.id = 'BBBBBB'
|
||||||
test_resource = Resource(submission, 'www.example.com/empty', '.jpeg')
|
test_resource = Resource(submission, 'www.example.com/empty', '.jpeg')
|
||||||
test_formatter = FileNameFormatter('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}')
|
test_formatter = FileNameFormatter('{REDDITOR}_{TITLE}_{POSTID}', '{SUBREDDIT}', 'ISO')
|
||||||
result = test_formatter.format_path(test_resource, tmp_path)
|
result = test_formatter.format_path(test_resource, tmp_path)
|
||||||
result.parent.mkdir(parents=True)
|
result.parent.mkdir(parents=True)
|
||||||
result.touch()
|
result.touch()
|
||||||
|
@ -237,7 +238,8 @@ def test_strip_emojies(test_string: str, expected: str):
|
||||||
))
|
))
|
||||||
def test_generate_dict_for_submission(test_submission_id: str, expected: dict, reddit_instance: praw.Reddit):
|
def test_generate_dict_for_submission(test_submission_id: str, expected: dict, reddit_instance: praw.Reddit):
|
||||||
test_submission = reddit_instance.submission(id=test_submission_id)
|
test_submission = reddit_instance.submission(id=test_submission_id)
|
||||||
result = FileNameFormatter._generate_name_dict_from_submission(test_submission)
|
test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
|
||||||
|
result = test_formatter._generate_name_dict_from_submission(test_submission)
|
||||||
assert all([result.get(key) == expected[key] for key in expected.keys()])
|
assert all([result.get(key) == expected[key] for key in expected.keys()])
|
||||||
|
|
||||||
|
|
||||||
|
@ -253,7 +255,8 @@ def test_generate_dict_for_submission(test_submission_id: str, expected: dict, r
|
||||||
))
|
))
|
||||||
def test_generate_dict_for_comment(test_comment_id: str, expected: dict, reddit_instance: praw.Reddit):
|
def test_generate_dict_for_comment(test_comment_id: str, expected: dict, reddit_instance: praw.Reddit):
|
||||||
test_comment = reddit_instance.comment(id=test_comment_id)
|
test_comment = reddit_instance.comment(id=test_comment_id)
|
||||||
result = FileNameFormatter._generate_name_dict_from_comment(test_comment)
|
test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
|
||||||
|
result = test_formatter._generate_name_dict_from_comment(test_comment)
|
||||||
assert all([result.get(key) == expected[key] for key in expected.keys()])
|
assert all([result.get(key) == expected[key] for key in expected.keys()])
|
||||||
|
|
||||||
|
|
||||||
|
@ -272,7 +275,7 @@ def test_format_archive_entry_comment(
|
||||||
reddit_instance: praw.Reddit,
|
reddit_instance: praw.Reddit,
|
||||||
):
|
):
|
||||||
test_comment = reddit_instance.comment(id=test_comment_id)
|
test_comment = reddit_instance.comment(id=test_comment_id)
|
||||||
test_formatter = FileNameFormatter(test_file_scheme, test_folder_scheme)
|
test_formatter = FileNameFormatter(test_file_scheme, test_folder_scheme, 'ISO')
|
||||||
test_entry = Resource(test_comment, '', '.json')
|
test_entry = Resource(test_comment, '', '.json')
|
||||||
result = test_formatter.format_path(test_entry, tmp_path)
|
result = test_formatter.format_path(test_entry, tmp_path)
|
||||||
assert result.name == expected_name
|
assert result.name == expected_name
|
||||||
|
@ -288,7 +291,7 @@ def test_multilevel_folder_scheme(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
submission: MagicMock,
|
submission: MagicMock,
|
||||||
):
|
):
|
||||||
test_formatter = FileNameFormatter('{POSTID}', test_folder_scheme)
|
test_formatter = FileNameFormatter('{POSTID}', test_folder_scheme, 'ISO')
|
||||||
test_resource = MagicMock()
|
test_resource = MagicMock()
|
||||||
test_resource.source_submission = submission
|
test_resource.source_submission = submission
|
||||||
test_resource.extension = '.png'
|
test_resource.extension = '.png'
|
||||||
|
@ -308,7 +311,8 @@ def test_multilevel_folder_scheme(
|
||||||
))
|
))
|
||||||
def test_preserve_emojis(test_name_string: str, expected: str, submission: MagicMock):
|
def test_preserve_emojis(test_name_string: str, expected: str, submission: MagicMock):
|
||||||
submission.title = test_name_string
|
submission.title = test_name_string
|
||||||
result = FileNameFormatter._format_name(submission, '{TITLE}')
|
test_formatter = FileNameFormatter('{TITLE}', '', 'ISO')
|
||||||
|
result = test_formatter._format_name(submission, '{TITLE}')
|
||||||
assert result == expected
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
@ -328,5 +332,18 @@ def test_convert_unicode_escapes(test_string: str, expected: str):
|
||||||
))
|
))
|
||||||
def test_convert_timestamp(test_datetime: datetime, expected: str):
|
def test_convert_timestamp(test_datetime: datetime, expected: str):
|
||||||
test_timestamp = test_datetime.timestamp()
|
test_timestamp = test_datetime.timestamp()
|
||||||
result = FileNameFormatter._convert_timestamp(test_timestamp)
|
test_formatter = FileNameFormatter('{POSTID}', '', 'ISO')
|
||||||
|
result = test_formatter._convert_timestamp(test_timestamp)
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(('test_time_format', 'expected'), (
|
||||||
|
('ISO', '2021-05-02T13:33:00'),
|
||||||
|
('%Y_%m', '2021_05'),
|
||||||
|
('%Y-%m-%d', '2021-05-02'),
|
||||||
|
))
|
||||||
|
def test_time_string_formats(test_time_format: str, expected: str):
|
||||||
|
test_time = datetime(2021, 5, 2, 13, 33)
|
||||||
|
test_formatter = FileNameFormatter('{TITLE}', '', test_time_format)
|
||||||
|
result = test_formatter._convert_timestamp(test_time.timestamp())
|
||||||
assert result == expected
|
assert result == expected
|
||||||
|
|
Loading…
Reference in a new issue