[models/utils] Track file offset for chunk parts

Preparation for smarter patching with downloader rewrite.
This commit is contained in:
derrod 2021-01-10 14:25:35 +01:00
parent a2280edea8
commit edad963200
3 changed files with 16 additions and 4 deletions

View file

@ -157,15 +157,18 @@ class JSONFML(FML):
_fm.chunk_parts = [] _fm.chunk_parts = []
_fm.install_tags = _fmj.pop('InstallTags', list()) _fm.install_tags = _fmj.pop('InstallTags', list())
_offset = 0
for _cpj in _fmj.pop('FileChunkParts'): for _cpj in _fmj.pop('FileChunkParts'):
_cp = ChunkPart() _cp = ChunkPart()
_cp.guid = guid_from_json(_cpj.pop('Guid')) _cp.guid = guid_from_json(_cpj.pop('Guid'))
_cp.offset = blob_to_num(_cpj.pop('Offset')) _cp.offset = blob_to_num(_cpj.pop('Offset'))
_cp.size = blob_to_num(_cpj.pop('Size')) _cp.size = blob_to_num(_cpj.pop('Size'))
_cp.file_offset = _offset
_fm.file_size += _cp.size _fm.file_size += _cp.size
if _cpj: if _cpj:
print(f'Non-read ChunkPart keys: {_cpj.keys()}') print(f'Non-read ChunkPart keys: {_cpj.keys()}')
_fm.chunk_parts.append(_cp) _fm.chunk_parts.append(_cp)
_offset += _cp.size
if _fmj: if _fmj:
print(f'Non-read FileManifest keys: {_fmj.keys()}') print(f'Non-read FileManifest keys: {_fmj.keys()}')

View file

@ -507,13 +507,20 @@ class FML:
# Each file is made up of "Chunk Parts" that can be spread across the "chunk stream" # Each file is made up of "Chunk Parts" that can be spread across the "chunk stream"
for fm in _fml.elements: for fm in _fml.elements:
_elem = struct.unpack('<I', bio.read(4))[0] _elem = struct.unpack('<I', bio.read(4))[0]
_offset = 0
for i in range(_elem): for i in range(_elem):
chunkp = ChunkPart() chunkp = ChunkPart()
_start = bio.tell()
_size = struct.unpack('<I', bio.read(4))[0] _size = struct.unpack('<I', bio.read(4))[0]
chunkp.guid = struct.unpack('<IIII', bio.read(16)) chunkp.guid = struct.unpack('<IIII', bio.read(16))
chunkp.offset = struct.unpack('<I', bio.read(4))[0] chunkp.offset = struct.unpack('<I', bio.read(4))[0]
chunkp.size = struct.unpack('<I', bio.read(4))[0] chunkp.size = struct.unpack('<I', bio.read(4))[0]
chunkp.file_offset = _offset
fm.chunk_parts.append(chunkp) fm.chunk_parts.append(chunkp)
_offset += chunkp.size
if (diff := (bio.tell() - _start - _size)) > 0:
logger.warning(f'Did not read {diff} bytes from chunk part!')
bio.seek(diff)
# we have to calculate the actual file size ourselves # we have to calculate the actual file size ourselves
for fm in _fml.elements: for fm in _fml.elements:
@ -601,10 +608,11 @@ class FileManifest:
class ChunkPart: class ChunkPart:
def __init__(self, guid=None, offset=0, size=0): def __init__(self, guid=None, offset=0, size=0, file_offset=0):
self.guid = guid self.guid = guid
self.offset = offset self.offset = offset
self.size = size self.size = size
self.file_offset = file_offset
# caches for things that are "expensive" to compute # caches for things that are "expensive" to compute
self._guid_str = None self._guid_str = None
self._guid_num = None self._guid_num = None
@ -623,8 +631,8 @@ class ChunkPart:
def __repr__(self): def __repr__(self):
guid_readable = '-'.join('{:08x}'.format(g) for g in self.guid) guid_readable = '-'.join('{:08x}'.format(g) for g in self.guid)
return '<ChunkPart (guid={}, offset={}, size={})>'.format( return '<ChunkPart (guid={}, offset={}, size={}, file_offset={})>'.format(
guid_readable, self.offset, self.size) guid_readable, self.offset, self.size, self.file_offset)
class CustomFields: class CustomFields:

View file

@ -123,7 +123,8 @@ class SaveGameHelper:
# create chunk part and write it to chunk buffer # create chunk part and write it to chunk buffer
cp = ChunkPart(guid=cur_chunk.guid, offset=cur_buffer.tell(), cp = ChunkPart(guid=cur_chunk.guid, offset=cur_buffer.tell(),
size=min(remaining, 1024 * 1024 - cur_buffer.tell())) size=min(remaining, 1024 * 1024 - cur_buffer.tell()),
file_offset=cf.tell())
_tmp = cf.read(cp.size) _tmp = cf.read(cp.size)
if not _tmp: if not _tmp:
self.log.warning(f'Got EOF for "{f.filename}" with {remaining} bytes remaining! ' self.log.warning(f'Got EOF for "{f.filename}" with {remaining} bytes remaining! '