CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-05-29 16:40:06 +12:00
Code Issues Releases Wiki Activity

Add shortcut in download for certain errors

Browse source
This commit is contained in:
Serene-Arc 2021-03-15 14:00:21 +10:00 committed by Ali Parlakci
parent 6aab009204
commit 3a093d0844
3 changed files with 12 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
bulkredditdownloader/downloader.py
View file

@ -335,7 +335,12 @@ class RedditDownloader:
if destination.exists():
logger.warning(f'File already exists: {destination}')
else:
res.download()
try:
res.download()
except errors.BulkDownloaderException:
logger.error(
f'Failed to download resource from {res.url} with downloader {downloader_class.__name__}')
return
if res.hash.hexdigest() in self.master_hash_list and self.args.no_dupes:
logger.warning(f'Resource from "{res.url}" and hash "{res.hash.hexdigest()}" downloaded elsewhere')
else:

3
bulkredditdownloader/resource.py
View file

@ -32,6 +32,9 @@ class Resource:
response = requests.get(url)
if response.status_code == 200:
return response.content
elif response.status_code in (301, 401, 403, 404):
logger.error(f'Unrecoverable error requesting resource: HTTP Code {response.status_code}')
return None
else:
raise requests.exceptions.ConnectionError
except requests.exceptions.ConnectionError:

4
bulkredditdownloader/tests/test_integration.py
View file

@ -22,6 +22,8 @@ from bulkredditdownloader.__main__ import cli
['-s', 'r/TrollXChromosomes/', '-L', 1],
['-s', 'TrollXChromosomes/', '-L', 1],
['-s', 'trollxchromosomes', '-L', 1],
['-s', 'trollxchromosomes,mindustry,python', '-L', 1],
['-s', 'trollxchromosomes, mindustry, python', '-L', 1],
['-s', 'trollxchromosomes', '-L', 1, '--time', 'day'],
['-s', 'trollxchromosomes', '-L', 1, '--sort', 'new'],
['-s', 'trollxchromosomes', '-L', 1, '--time', 'day', '--sort', 'new'],
@ -46,13 +48,13 @@ def test_cli_download_subreddits(test_args: list[str], tmp_path: Path):
['-l', 'https://www.reddit.com/r/TrollXChromosomes/comments/m2601g/its_a_step_in_the_right_direction/'],
['-l', 'm3hxzd'], # Really long title used to overflow filename limit
['-l', 'm3kua3'], # Has a deleted user
['-l', 'm5bqkf'], # Resource leading to a 404
))
def test_cli_download_links(test_args: list[str], tmp_path: Path):
runner = CliRunner()
test_args = ['download', str(tmp_path), '-v', '--config', 'test_config.cfg'] + test_args
result = runner.invoke(cli, test_args)
assert result.exit_code == 0
assert len(list(tmp_path.iterdir())) == 1
@pytest.mark.online