CodeMirror/legendary
1
0
Fork 0
mirror of https://github.com/derrod/legendary.git synced 2024-06-27 02:30:30 +12:00
Code Issues Projects Releases Wiki Activity

[downloader/models] Minor code cleanups

Browse source
This commit is contained in:
derrod 2020-04-25 10:18:45 +02:00
parent f1885639ae
commit 96602d1890
2 changed files with 10 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
legendary/downloader/manager.py
View file

@ -86,7 +86,7 @@ class DLManager(Process):
c_guid = self.chunks_to_dl.popleft() c_guid = self.chunks_to_dl.popleft()
chunk = self.chunk_data_list.get_chunk_by_guid(c_guid) chunk = self.chunk_data_list.get_chunk_by_guid(c_guid)
self.log.debug(f'Adding {chunk.guid_str} (active: {self.active_tasks})') self.log.debug(f'Adding {chunk.guid_num} (active: {self.active_tasks})')
try: try:
self.dl_worker_queue.put(DownloaderTask(url=self.base_url + '/' + chunk.path, self.dl_worker_queue.put(DownloaderTask(url=self.base_url + '/' + chunk.path,
chunk_guid=c_guid, shm=sms), chunk_guid=c_guid, shm=sms),
@ -147,11 +147,9 @@ class DLManager(Process):
continue continue
while (task.chunk_guid in in_buffer) or task.chunk_file: while (task.chunk_guid in in_buffer) or task.chunk_file:
if task.chunk_file: # re-using from an old file res_shm = None
res_shm = None if not task.chunk_file: # not re-using from an old file
else: res_shm = in_buffer[task.chunk_guid].shm
res = in_buffer[task.chunk_guid]
res_shm = res.shm
try: try:
self.writer_queue.put(WriterTask( self.writer_queue.put(WriterTask(
@ -194,7 +192,7 @@ class DLManager(Process):
self.active_tasks += 1 self.active_tasks += 1
except Exception as e: except Exception as e:
self.log.warning(f'Failed adding retry task to queue! {e!r}') self.log.warning(f'Failed adding retry task to queue! {e!r}')
# if no reserved memory, add to the beginning of the normal queue # If this failed for whatever reason, put the chunk at the front of the DL list
self.chunks_to_dl.appendleft(res.chunk_guid) self.chunks_to_dl.appendleft(res.chunk_guid)
except Empty: except Empty:
pass pass
@ -351,10 +349,10 @@ class DLManager(Process):
for next_chunk in chunkstream_starts: for next_chunk in chunkstream_starts:
self.log.debug(f'- Chunkstream start: {next_chunk!r}') self.log.debug(f'- Chunkstream start: {next_chunk!r}')
while file_list := chunk_to_file_map.get(next_chunk.guid_num): while file_deque := chunk_to_file_map.get(next_chunk.guid_num):
current_file = file_list.popleft() current_file = file_deque.popleft()
if len(file_list) == 0: if len(file_deque) == 0:
del chunk_to_file_map[next_chunk.guid_num] del chunk_to_file_map[next_chunk.guid_num]
# skip unchanged files # skip unchanged files
@ -474,8 +472,7 @@ class DLManager(Process):
# create the shared memory segments and add them to their respective pools # create the shared memory segments and add them to their respective pools
for i in range(int(self.shared_memory.size / self.analysis.biggest_chunk)): for i in range(int(self.shared_memory.size / self.analysis.biggest_chunk)):
_sms = SharedMemorySegment(offset=i * self.analysis.biggest_chunk, _sms = SharedMemorySegment(offset=i * self.analysis.biggest_chunk,
end=i * self.analysis.biggest_chunk + self.analysis.biggest_chunk, end=i * self.analysis.biggest_chunk + self.analysis.biggest_chunk)
_id=i)
self.sms.append(_sms) self.sms.append(_sms)
self.log.debug(f'Created {len(self.sms)} shared memory segments.') self.log.debug(f'Created {len(self.sms)} shared memory segments.')

3
legendary/models/downloading.py
View file

@ -85,10 +85,9 @@ class SharedMemorySegment:
Segment of the shared memory used for one Chunk Segment of the shared memory used for one Chunk
""" """
def __init__(self, offset=0, end=1024 * 1024, _id=None): def __init__(self, offset=0, end=1024 * 1024):
self.offset = offset self.offset = offset
self.end = end self.end = end
self._id = _id
@property @property
def size(self): def size(self):