LaunchDialog: Add a middle-ground solution for concurrent image downloads
This commit is contained in:
parent
681b3013ad
commit
cf5fd415e5
3 changed files with 62 additions and 22 deletions
1
.github/workflows/tests.yml
vendored
1
.github/workflows/tests.yml
vendored
|
@ -88,7 +88,6 @@ jobs:
|
||||||
python-version: '3.9'
|
python-version: '3.9'
|
||||||
- name: Dependencies
|
- name: Dependencies
|
||||||
run: |
|
run: |
|
||||||
pip3 install certifi==2022.6.15
|
|
||||||
pip3 install -r requirements.txt
|
pip3 install -r requirements.txt
|
||||||
pip3 install -r requirements-presence.txt
|
pip3 install -r requirements-presence.txt
|
||||||
- name: Build Dependencies
|
- name: Build Dependencies
|
||||||
|
|
|
@ -2,8 +2,9 @@ import os
|
||||||
import platform
|
import platform
|
||||||
from logging import getLogger
|
from logging import getLogger
|
||||||
|
|
||||||
from PyQt5.QtCore import Qt, pyqtSignal, QRunnable, QObject, QThreadPool, QSettings, pyqtSlot, QCoreApplication
|
from PyQt5.QtCore import Qt, pyqtSignal, QRunnable, QObject, QThreadPool, QSettings, pyqtSlot
|
||||||
from PyQt5.QtWidgets import QDialog, QApplication
|
from PyQt5.QtWidgets import QDialog, QApplication
|
||||||
|
from legendary.models.game import Game
|
||||||
from requests.exceptions import ConnectionError, HTTPError
|
from requests.exceptions import ConnectionError, HTTPError
|
||||||
|
|
||||||
from rare.components.dialogs.login import LoginDialog
|
from rare.components.dialogs.login import LoginDialog
|
||||||
|
@ -29,19 +30,41 @@ class LaunchWorker(QRunnable):
|
||||||
self.signals = LaunchWorker.Signals()
|
self.signals = LaunchWorker.Signals()
|
||||||
self.core = LegendaryCoreSingleton()
|
self.core = LegendaryCoreSingleton()
|
||||||
|
|
||||||
def run(self):
|
def run_real(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
self.run_real()
|
||||||
|
self.signals.deleteLater()
|
||||||
|
|
||||||
|
|
||||||
class ImageWorker(LaunchWorker):
|
class ImageWorker(LaunchWorker):
|
||||||
|
# FIXME: this is a middle-ground solution for concurrent downloads
|
||||||
|
class DownloadSlot(QObject):
|
||||||
|
def __init__(self, signals: LaunchWorker.Signals):
|
||||||
|
super(ImageWorker.DownloadSlot, self).__init__()
|
||||||
|
self.signals = signals
|
||||||
|
self.counter = 0
|
||||||
|
self.length = 0
|
||||||
|
|
||||||
|
@pyqtSlot(object)
|
||||||
|
def counter_inc(self, game: Game):
|
||||||
|
self.signals.progress.emit(
|
||||||
|
int(self.counter / self.length * 50),
|
||||||
|
self.tr("Downloading image for <b>{}</b>").format(game.app_title)
|
||||||
|
)
|
||||||
|
self.counter += 1
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super(ImageWorker, self).__init__()
|
super(ImageWorker, self).__init__()
|
||||||
|
# FIXME: this is a middle-ground solution for concurrent downloads
|
||||||
|
self.dl_slot = ImageWorker.DownloadSlot(self.signals)
|
||||||
self.image_manager = ImageManagerSingleton()
|
self.image_manager = ImageManagerSingleton()
|
||||||
|
|
||||||
def tr(self, t) -> str:
|
def tr(self, t) -> str:
|
||||||
return QCoreApplication.translate(self.__class__.__name__, t)
|
return QApplication.instance().translate(self.__class__.__name__, t)
|
||||||
|
|
||||||
def run(self):
|
def run_real(self):
|
||||||
# Download Images
|
# Download Images
|
||||||
games, dlcs = self.core.get_game_and_dlc_list(update_assets=True, skip_ue=False)
|
games, dlcs = self.core.get_game_and_dlc_list(update_assets=True, skip_ue=False)
|
||||||
self.signals.result.emit((games, dlcs), "gamelist")
|
self.signals.result.emit((games, dlcs), "gamelist")
|
||||||
|
@ -53,20 +76,21 @@ class ImageWorker(LaunchWorker):
|
||||||
|
|
||||||
game_list = games + dlc_list + na_games + na_dlc_list
|
game_list = games + dlc_list + na_games + na_dlc_list
|
||||||
|
|
||||||
|
self.dl_slot.length = len(game_list)
|
||||||
for i, game in enumerate(game_list):
|
for i, game in enumerate(game_list):
|
||||||
if game.app_title == "Unreal Engine":
|
if game.app_title == "Unreal Engine":
|
||||||
game.app_title += f" {game.app_name.split('_')[-1]}"
|
game.app_title += f" {game.app_name.split('_')[-1]}"
|
||||||
self.core.lgd.set_game_meta(game.app_name, game)
|
self.core.lgd.set_game_meta(game.app_name, game)
|
||||||
self.signals.progress.emit(
|
# self.image_manager.download_image_blocking(game)
|
||||||
int(i / len(game_list) * 50),
|
self.image_manager.download_image(game, self.dl_slot.counter_inc, priority=0)
|
||||||
self.tr("Downloading image for <b>{}</b>").format(game.app_title)
|
# FIXME: this is a middle-ground solution for concurrent downloads
|
||||||
)
|
while self.dl_slot.counter < len(game_list):
|
||||||
self.image_manager.download_image_blocking(game)
|
QApplication.instance().processEvents()
|
||||||
|
self.dl_slot.deleteLater()
|
||||||
|
|
||||||
igame_list = self.core.get_installed_list(include_dlc=True)
|
igame_list = self.core.get_installed_list(include_dlc=True)
|
||||||
|
|
||||||
# FIXME: incorporate installed game status checking here for now, still slow
|
# FIXME: incorporate installed game status checking here for now, still slow
|
||||||
# if igame := self.core.get_installed_game(game.app_name, skip_sync=True):
|
|
||||||
for i, igame in enumerate(igame_list):
|
for i, igame in enumerate(igame_list):
|
||||||
self.signals.progress.emit(
|
self.signals.progress.emit(
|
||||||
int(i / len(igame_list) * 50) + 50,
|
int(i / len(igame_list) * 50) + 50,
|
||||||
|
@ -100,7 +124,7 @@ class ApiRequestWorker(LaunchWorker):
|
||||||
super(ApiRequestWorker, self).__init__()
|
super(ApiRequestWorker, self).__init__()
|
||||||
self.settings = QSettings()
|
self.settings = QSettings()
|
||||||
|
|
||||||
def run(self) -> None:
|
def run_real(self) -> None:
|
||||||
if self.settings.value("mac_meta", platform.system() == "Darwin", bool):
|
if self.settings.value("mac_meta", platform.system() == "Darwin", bool):
|
||||||
try:
|
try:
|
||||||
result = self.core.get_game_and_dlc_list(update_assets=False, platform="Mac")
|
result = self.core.get_game_and_dlc_list(update_assets=False, platform="Mac")
|
||||||
|
|
|
@ -1,9 +1,8 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import pickle
|
import pickle
|
||||||
import zlib
|
import zlib
|
||||||
|
# from concurrent import futures
|
||||||
from logging import getLogger
|
from logging import getLogger
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
@ -30,6 +29,8 @@ from rare.lgndr.core import LegendaryCore
|
||||||
from rare.models.signals import GlobalSignals
|
from rare.models.signals import GlobalSignals
|
||||||
from rare.utils.paths import image_dir, resources_path
|
from rare.utils.paths import image_dir, resources_path
|
||||||
|
|
||||||
|
# from requests_futures.sessions import FuturesSession
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@ -86,8 +87,8 @@ class ImageSize:
|
||||||
class ImageManager(QObject):
|
class ImageManager(QObject):
|
||||||
class Worker(QRunnable):
|
class Worker(QRunnable):
|
||||||
class Signals(QObject):
|
class Signals(QObject):
|
||||||
# str: app_name
|
# object: Game
|
||||||
completed = pyqtSignal(str)
|
completed = pyqtSignal(object)
|
||||||
|
|
||||||
def __init__(self, func: Callable, updates: List, json_data: Dict, game: Game):
|
def __init__(self, func: Callable, updates: List, json_data: Dict, game: Game):
|
||||||
super(ImageManager.Worker, self).__init__()
|
super(ImageManager.Worker, self).__init__()
|
||||||
|
@ -101,7 +102,7 @@ class ImageManager(QObject):
|
||||||
def run(self):
|
def run(self):
|
||||||
self.func(self.updates, self.json_data, self.game)
|
self.func(self.updates, self.json_data, self.game)
|
||||||
logger.debug(f" Emitting singal for game {self.game.app_name} - {self.game.app_title}")
|
logger.debug(f" Emitting singal for game {self.game.app_name} - {self.game.app_title}")
|
||||||
self.signals.completed.emit(self.game.app_name)
|
self.signals.completed.emit(self.game)
|
||||||
|
|
||||||
def __init__(self, signals: GlobalSignals, core: LegendaryCore):
|
def __init__(self, signals: GlobalSignals, core: LegendaryCore):
|
||||||
# lk: the ordering in __img_types matters for the order of fallbacks
|
# lk: the ordering in __img_types matters for the order of fallbacks
|
||||||
|
@ -179,7 +180,7 @@ class ImageManager(QObject):
|
||||||
|
|
||||||
return updates, json_data
|
return updates, json_data
|
||||||
|
|
||||||
def __download(self, updates, json_data, game) -> bool:
|
def __download(self, updates, json_data, game, use_async: bool = False) -> bool:
|
||||||
# Decompress existing image.cache
|
# Decompress existing image.cache
|
||||||
if not self.__img_cache(game.app_name).is_file():
|
if not self.__img_cache(game.app_name).is_file():
|
||||||
cache_data = dict(zip(self.__img_types, [None] * len(self.__img_types)))
|
cache_data = dict(zip(self.__img_types, [None] * len(self.__img_types)))
|
||||||
|
@ -193,6 +194,22 @@ class ImageManager(QObject):
|
||||||
if cache_data[image["type"]] is None or json_data[image["type"]] != image["md5"]
|
if cache_data[image["type"]] is None or json_data[image["type"]] != image["md5"]
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Download
|
||||||
|
# # lk: Keep this so I don't have to go looking for it again,
|
||||||
|
# # lk: it might be useful in the future.
|
||||||
|
# if use_async and len(updates) > 1:
|
||||||
|
# session = FuturesSession(max_workers=len(self.__img_types))
|
||||||
|
# image_requests = []
|
||||||
|
# for image in updates:
|
||||||
|
# logger.info(f"Downloading {image['type']} for {game.app_title}")
|
||||||
|
# json_data[image["type"]] = image["md5"]
|
||||||
|
# payload = {"resize": 1, "w": ImageSize.Image.size.width(), "h": ImageSize.Image.size.height()}
|
||||||
|
# req = session.get(image["url"], params=payload)
|
||||||
|
# req.image_type = image["type"]
|
||||||
|
# image_requests.append(req)
|
||||||
|
# for req in futures.as_completed(image_requests):
|
||||||
|
# cache_data[req.image_type] = req.result().content
|
||||||
|
# else:
|
||||||
for image in updates:
|
for image in updates:
|
||||||
logger.info(f"Downloading {image['type']} for {game.app_title}")
|
logger.info(f"Downloading {image['type']} for {game.app_title}")
|
||||||
json_data[image["type"]] = image["md5"]
|
json_data[image["type"]] = image["md5"]
|
||||||
|
@ -291,18 +308,18 @@ class ImageManager(QObject):
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def download_image(
|
def download_image(
|
||||||
self, game: Game, load_callback: Callable[[], None], priority: int, force: bool = False
|
self, game: Game, load_callback: Callable[[Game], None], priority: int, force: bool = False
|
||||||
) -> None:
|
) -> None:
|
||||||
updates, json_data = self.__prepare_download(game, force)
|
updates, json_data = self.__prepare_download(game, force)
|
||||||
if not updates:
|
if not updates:
|
||||||
load_callback()
|
load_callback(game)
|
||||||
return
|
return
|
||||||
if updates and game.app_name not in self.__worker_app_names:
|
if updates and game.app_name not in self.__worker_app_names:
|
||||||
image_worker = ImageManager.Worker(self.__download, updates, json_data, game)
|
image_worker = ImageManager.Worker(self.__download, updates, json_data, game)
|
||||||
self.__worker_app_names.append(game.app_name)
|
self.__worker_app_names.append(game.app_name)
|
||||||
|
|
||||||
image_worker.signals.completed.connect(load_callback)
|
image_worker.signals.completed.connect(load_callback)
|
||||||
image_worker.signals.completed.connect(lambda app_name: self.__worker_app_names.remove(app_name))
|
image_worker.signals.completed.connect(lambda g: self.__worker_app_names.remove(g.app_name))
|
||||||
self.threadpool.start(image_worker, priority)
|
self.threadpool.start(image_worker, priority)
|
||||||
|
|
||||||
def download_image_blocking(self, game: Game, force: bool = False) -> None:
|
def download_image_blocking(self, game: Game, force: bool = False) -> None:
|
||||||
|
@ -310,7 +327,7 @@ class ImageManager(QObject):
|
||||||
if not updates:
|
if not updates:
|
||||||
return
|
return
|
||||||
if updates:
|
if updates:
|
||||||
self.__download(updates, json_data, game)
|
self.__download(updates, json_data, game, use_async=True)
|
||||||
|
|
||||||
def __get_cover(
|
def __get_cover(
|
||||||
self, container: Union[Type[QPixmap], Type[QImage]], app_name: str, color: bool = True
|
self, container: Union[Type[QPixmap], Type[QImage]], app_name: str, color: bool = True
|
||||||
|
|
Loading…
Reference in a new issue