Merge pull request #425 from Serene-Arc/bug_fix_424
Fix download retries logic
This commit is contained in:
commit
0b5f18430f
1 changed files with 6 additions and 6 deletions
|
@ -28,8 +28,7 @@ class Resource:
|
||||||
self.extension = self._determine_extension()
|
self.extension = self._determine_extension()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def retry_download(url: str, max_wait_time: int) -> Optional[bytes]:
|
def retry_download(url: str, max_wait_time: int, current_wait_time: int = 60) -> Optional[bytes]:
|
||||||
wait_time = 60
|
|
||||||
try:
|
try:
|
||||||
response = requests.get(url)
|
response = requests.get(url)
|
||||||
if re.match(r'^2\d{2}', str(response.status_code)) and response.content:
|
if re.match(r'^2\d{2}', str(response.status_code)) and response.content:
|
||||||
|
@ -40,10 +39,11 @@ class Resource:
|
||||||
raise BulkDownloaderException(
|
raise BulkDownloaderException(
|
||||||
f'Unrecoverable error requesting resource: HTTP Code {response.status_code}')
|
f'Unrecoverable error requesting resource: HTTP Code {response.status_code}')
|
||||||
except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e:
|
except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e:
|
||||||
logger.warning(f'Error occured downloading from {url}, waiting {wait_time} seconds: {e}')
|
logger.warning(f'Error occured downloading from {url}, waiting {current_wait_time} seconds: {e}')
|
||||||
time.sleep(wait_time)
|
time.sleep(current_wait_time)
|
||||||
if wait_time < max_wait_time:
|
if current_wait_time < max_wait_time:
|
||||||
return Resource.retry_download(url, max_wait_time)
|
current_wait_time += 60
|
||||||
|
return Resource.retry_download(url, max_wait_time, current_wait_time)
|
||||||
else:
|
else:
|
||||||
logger.error(f'Max wait time exceeded for resource at url {url}')
|
logger.error(f'Max wait time exceeded for resource at url {url}')
|
||||||
raise
|
raise
|
||||||
|
|
Loading…
Reference in a new issue