1
0
Fork 0
mirror of synced 2024-06-29 03:31:06 +12:00

LaunchDialog: Add a middle-ground solution for concurrent image downloads

This commit is contained in:
loathingKernel 2022-09-11 14:51:37 +03:00
parent 681b3013ad
commit cf5fd415e5
3 changed files with 62 additions and 22 deletions

View file

@ -88,7 +88,6 @@ jobs:
python-version: '3.9' python-version: '3.9'
- name: Dependencies - name: Dependencies
run: | run: |
pip3 install certifi==2022.6.15
pip3 install -r requirements.txt pip3 install -r requirements.txt
pip3 install -r requirements-presence.txt pip3 install -r requirements-presence.txt
- name: Build Dependencies - name: Build Dependencies

View file

@ -2,8 +2,9 @@ import os
import platform import platform
from logging import getLogger from logging import getLogger
from PyQt5.QtCore import Qt, pyqtSignal, QRunnable, QObject, QThreadPool, QSettings, pyqtSlot, QCoreApplication from PyQt5.QtCore import Qt, pyqtSignal, QRunnable, QObject, QThreadPool, QSettings, pyqtSlot
from PyQt5.QtWidgets import QDialog, QApplication from PyQt5.QtWidgets import QDialog, QApplication
from legendary.models.game import Game
from requests.exceptions import ConnectionError, HTTPError from requests.exceptions import ConnectionError, HTTPError
from rare.components.dialogs.login import LoginDialog from rare.components.dialogs.login import LoginDialog
@ -29,19 +30,41 @@ class LaunchWorker(QRunnable):
self.signals = LaunchWorker.Signals() self.signals = LaunchWorker.Signals()
self.core = LegendaryCoreSingleton() self.core = LegendaryCoreSingleton()
def run(self): def run_real(self):
pass pass
def run(self):
self.run_real()
self.signals.deleteLater()
class ImageWorker(LaunchWorker): class ImageWorker(LaunchWorker):
# FIXME: this is a middle-ground solution for concurrent downloads
class DownloadSlot(QObject):
def __init__(self, signals: LaunchWorker.Signals):
super(ImageWorker.DownloadSlot, self).__init__()
self.signals = signals
self.counter = 0
self.length = 0
@pyqtSlot(object)
def counter_inc(self, game: Game):
self.signals.progress.emit(
int(self.counter / self.length * 50),
self.tr("Downloading image for <b>{}</b>").format(game.app_title)
)
self.counter += 1
def __init__(self): def __init__(self):
super(ImageWorker, self).__init__() super(ImageWorker, self).__init__()
# FIXME: this is a middle-ground solution for concurrent downloads
self.dl_slot = ImageWorker.DownloadSlot(self.signals)
self.image_manager = ImageManagerSingleton() self.image_manager = ImageManagerSingleton()
def tr(self, t) -> str: def tr(self, t) -> str:
return QCoreApplication.translate(self.__class__.__name__, t) return QApplication.instance().translate(self.__class__.__name__, t)
def run(self): def run_real(self):
# Download Images # Download Images
games, dlcs = self.core.get_game_and_dlc_list(update_assets=True, skip_ue=False) games, dlcs = self.core.get_game_and_dlc_list(update_assets=True, skip_ue=False)
self.signals.result.emit((games, dlcs), "gamelist") self.signals.result.emit((games, dlcs), "gamelist")
@ -53,20 +76,21 @@ class ImageWorker(LaunchWorker):
game_list = games + dlc_list + na_games + na_dlc_list game_list = games + dlc_list + na_games + na_dlc_list
self.dl_slot.length = len(game_list)
for i, game in enumerate(game_list): for i, game in enumerate(game_list):
if game.app_title == "Unreal Engine": if game.app_title == "Unreal Engine":
game.app_title += f" {game.app_name.split('_')[-1]}" game.app_title += f" {game.app_name.split('_')[-1]}"
self.core.lgd.set_game_meta(game.app_name, game) self.core.lgd.set_game_meta(game.app_name, game)
self.signals.progress.emit( # self.image_manager.download_image_blocking(game)
int(i / len(game_list) * 50), self.image_manager.download_image(game, self.dl_slot.counter_inc, priority=0)
self.tr("Downloading image for <b>{}</b>").format(game.app_title) # FIXME: this is a middle-ground solution for concurrent downloads
) while self.dl_slot.counter < len(game_list):
self.image_manager.download_image_blocking(game) QApplication.instance().processEvents()
self.dl_slot.deleteLater()
igame_list = self.core.get_installed_list(include_dlc=True) igame_list = self.core.get_installed_list(include_dlc=True)
# FIXME: incorporate installed game status checking here for now, still slow # FIXME: incorporate installed game status checking here for now, still slow
# if igame := self.core.get_installed_game(game.app_name, skip_sync=True):
for i, igame in enumerate(igame_list): for i, igame in enumerate(igame_list):
self.signals.progress.emit( self.signals.progress.emit(
int(i / len(igame_list) * 50) + 50, int(i / len(igame_list) * 50) + 50,
@ -100,7 +124,7 @@ class ApiRequestWorker(LaunchWorker):
super(ApiRequestWorker, self).__init__() super(ApiRequestWorker, self).__init__()
self.settings = QSettings() self.settings = QSettings()
def run(self) -> None: def run_real(self) -> None:
if self.settings.value("mac_meta", platform.system() == "Darwin", bool): if self.settings.value("mac_meta", platform.system() == "Darwin", bool):
try: try:
result = self.core.get_game_and_dlc_list(update_assets=False, platform="Mac") result = self.core.get_game_and_dlc_list(update_assets=False, platform="Mac")

View file

@ -1,9 +1,8 @@
from __future__ import annotations
import hashlib import hashlib
import json import json
import pickle import pickle
import zlib import zlib
# from concurrent import futures
from logging import getLogger from logging import getLogger
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
@ -30,6 +29,8 @@ from rare.lgndr.core import LegendaryCore
from rare.models.signals import GlobalSignals from rare.models.signals import GlobalSignals
from rare.utils.paths import image_dir, resources_path from rare.utils.paths import image_dir, resources_path
# from requests_futures.sessions import FuturesSession
if TYPE_CHECKING: if TYPE_CHECKING:
pass pass
@ -86,8 +87,8 @@ class ImageSize:
class ImageManager(QObject): class ImageManager(QObject):
class Worker(QRunnable): class Worker(QRunnable):
class Signals(QObject): class Signals(QObject):
# str: app_name # object: Game
completed = pyqtSignal(str) completed = pyqtSignal(object)
def __init__(self, func: Callable, updates: List, json_data: Dict, game: Game): def __init__(self, func: Callable, updates: List, json_data: Dict, game: Game):
super(ImageManager.Worker, self).__init__() super(ImageManager.Worker, self).__init__()
@ -101,7 +102,7 @@ class ImageManager(QObject):
def run(self): def run(self):
self.func(self.updates, self.json_data, self.game) self.func(self.updates, self.json_data, self.game)
logger.debug(f" Emitting singal for game {self.game.app_name} - {self.game.app_title}") logger.debug(f" Emitting singal for game {self.game.app_name} - {self.game.app_title}")
self.signals.completed.emit(self.game.app_name) self.signals.completed.emit(self.game)
def __init__(self, signals: GlobalSignals, core: LegendaryCore): def __init__(self, signals: GlobalSignals, core: LegendaryCore):
# lk: the ordering in __img_types matters for the order of fallbacks # lk: the ordering in __img_types matters for the order of fallbacks
@ -179,7 +180,7 @@ class ImageManager(QObject):
return updates, json_data return updates, json_data
def __download(self, updates, json_data, game) -> bool: def __download(self, updates, json_data, game, use_async: bool = False) -> bool:
# Decompress existing image.cache # Decompress existing image.cache
if not self.__img_cache(game.app_name).is_file(): if not self.__img_cache(game.app_name).is_file():
cache_data = dict(zip(self.__img_types, [None] * len(self.__img_types))) cache_data = dict(zip(self.__img_types, [None] * len(self.__img_types)))
@ -193,6 +194,22 @@ class ImageManager(QObject):
if cache_data[image["type"]] is None or json_data[image["type"]] != image["md5"] if cache_data[image["type"]] is None or json_data[image["type"]] != image["md5"]
] ]
# Download
# # lk: Keep this so I don't have to go looking for it again,
# # lk: it might be useful in the future.
# if use_async and len(updates) > 1:
# session = FuturesSession(max_workers=len(self.__img_types))
# image_requests = []
# for image in updates:
# logger.info(f"Downloading {image['type']} for {game.app_title}")
# json_data[image["type"]] = image["md5"]
# payload = {"resize": 1, "w": ImageSize.Image.size.width(), "h": ImageSize.Image.size.height()}
# req = session.get(image["url"], params=payload)
# req.image_type = image["type"]
# image_requests.append(req)
# for req in futures.as_completed(image_requests):
# cache_data[req.image_type] = req.result().content
# else:
for image in updates: for image in updates:
logger.info(f"Downloading {image['type']} for {game.app_title}") logger.info(f"Downloading {image['type']} for {game.app_title}")
json_data[image["type"]] = image["md5"] json_data[image["type"]] = image["md5"]
@ -291,18 +308,18 @@ class ImageManager(QObject):
return data return data
def download_image( def download_image(
self, game: Game, load_callback: Callable[[], None], priority: int, force: bool = False self, game: Game, load_callback: Callable[[Game], None], priority: int, force: bool = False
) -> None: ) -> None:
updates, json_data = self.__prepare_download(game, force) updates, json_data = self.__prepare_download(game, force)
if not updates: if not updates:
load_callback() load_callback(game)
return return
if updates and game.app_name not in self.__worker_app_names: if updates and game.app_name not in self.__worker_app_names:
image_worker = ImageManager.Worker(self.__download, updates, json_data, game) image_worker = ImageManager.Worker(self.__download, updates, json_data, game)
self.__worker_app_names.append(game.app_name) self.__worker_app_names.append(game.app_name)
image_worker.signals.completed.connect(load_callback) image_worker.signals.completed.connect(load_callback)
image_worker.signals.completed.connect(lambda app_name: self.__worker_app_names.remove(app_name)) image_worker.signals.completed.connect(lambda g: self.__worker_app_names.remove(g.app_name))
self.threadpool.start(image_worker, priority) self.threadpool.start(image_worker, priority)
def download_image_blocking(self, game: Game, force: bool = False) -> None: def download_image_blocking(self, game: Game, force: bool = False) -> None:
@ -310,7 +327,7 @@ class ImageManager(QObject):
if not updates: if not updates:
return return
if updates: if updates:
self.__download(updates, json_data, game) self.__download(updates, json_data, game, use_async=True)
def __get_cover( def __get_cover(
self, container: Union[Type[QPixmap], Type[QImage]], app_name: str, color: bool = True self, container: Union[Type[QPixmap], Type[QImage]], app_name: str, color: bool = True