From 99bcdbaaf069b1c149100607eb663412c75a4321 Mon Sep 17 00:00:00 2001 From: loathingKernel <142770+loathingKernel@users.noreply.github.com> Date: Mon, 27 May 2024 15:33:44 +0300 Subject: [PATCH] ImageManager: Download and prepare wide images too --- rare/shared/image_manager.py | 273 +++++++++++++++++++++-------------- 1 file changed, 166 insertions(+), 107 deletions(-) diff --git a/rare/shared/image_manager.py b/rare/shared/image_manager.py index edd5e448..3d9404ae 100644 --- a/rare/shared/image_manager.py +++ b/rare/shared/image_manager.py @@ -2,6 +2,7 @@ import hashlib import json import pickle import zlib + # from concurrent import futures from logging import getLogger from pathlib import Path @@ -9,25 +10,8 @@ from typing import TYPE_CHECKING, Optional, Set from typing import Tuple, Dict, Union, Type, List, Callable import requests -from PyQt5.QtCore import ( - Qt, - pyqtSignal, - QObject, - QSize, - QThreadPool, - QRunnable, - QRect, - QRectF, -) -from PyQt5.QtGui import ( - QPixmap, - QImage, - QPainter, - QPainterPath, - QBrush, - QTransform, - QPen, -) +from PyQt5.QtCore import Qt, pyqtSignal, QObject, QSize, QThreadPool, QRunnable, QRect, QRectF, pyqtSlot +from PyQt5.QtGui import QPixmap, QImage, QPainter, QPainterPath, QBrush, QTransform, QPen from PyQt5.QtWidgets import QApplication from legendary.models.game import Game @@ -66,8 +50,12 @@ class ImageManager(QObject): def __init__(self, signals: GlobalSignals, core: LegendaryCore): # lk: the ordering in __img_types matters for the order of fallbacks - # self.__img_types: Tuple = ("DieselGameBoxTall", "Thumbnail", "DieselGameBoxLogo", "DieselGameBox", "OfferImageTall") - self.__img_tall_types: Tuple = ("DieselGameBoxTall", "Thumbnail", "DieselGameBoxLogo", "OfferImageTall") + # {'AndroidIcon', 'DieselGameBox', 'DieselGameBoxLogo', 'DieselGameBoxTall', 'DieselGameBoxWide', + # 'ESRB', 'Featured', 'OfferImageTall', 'OfferImageWide', 'Screenshot', 'Thumbnail'} + self.__img_tall_types: Tuple = ("DieselGameBoxTall", "OfferImageTall", "Thumbnail") + self.__img_wide_types: Tuple = ("DieselGameBoxWide", "DieselGameBox", "OfferImageWide", "Screenshot") + self.__img_logo_types: Tuple = ("DieselGameBoxLogo",) + self.__img_types: Tuple = self.__img_tall_types + self.__img_wide_types + self.__img_logo_types self.__dl_retries = 1 self.__worker_app_names: Set[str] = set() super(QObject, self).__init__() @@ -82,7 +70,7 @@ class ImageManager(QObject): self.device = ImageSize.Preset(1, QApplication.instance().devicePixelRatio()) self.threadpool = QThreadPool() - self.threadpool.setMaxThreadCount(4) + self.threadpool.setMaxThreadCount(6) def __img_dir(self, app_name: str) -> Path: return self.image_dir.joinpath(app_name) @@ -94,100 +82,144 @@ class ImageManager(QObject): return self.__img_dir(app_name).joinpath("image.cache") def __img_tall_color(self, app_name: str) -> Path: - return self.__img_dir(app_name).joinpath("tall_installed.png") + return self.__img_dir(app_name).joinpath("tall.png") def __img_tall_gray(self, app_name: str) -> Path: return self.__img_dir(app_name).joinpath("tall_uninstalled.png") def __img_wide_color(self, app_name: str) -> Path: - return self.__img_dir(app_name).joinpath("wide_installed.png") + return self.__img_dir(app_name).joinpath("wide.png") def __img_wide_gray(self, app_name: str) -> Path: return self.__img_dir(app_name).joinpath("wide_uninstalled.png") - def __img_desktop_icon(self, app_name: str) -> Path: + def __img_logo(self, app_name: str) -> Path: + return self.__img_dir(app_name).joinpath("logo.png") + + def __img_icon(self, app_name: str) -> Path: return self.__img_dir(app_name).joinpath(f"icon.{desktop_icon_suffix()}") + def __img_all(self, app_name: str) -> Tuple: + return ( + self.__img_icon(app_name), + self.__img_tall_color(app_name), + self.__img_tall_gray(app_name), + self.__img_wide_color(app_name), + self.__img_tall_gray(app_name), + ) + def __prepare_download(self, game: Game, force: bool = False) -> Tuple[List, Dict]: if force and self.__img_dir(game.app_name).exists(): - self.__img_desktop_icon(game.app_name).unlink(missing_ok=True) - self.__img_tall_color(game.app_name).unlink(missing_ok=True) - self.__img_tall_gray(game.app_name).unlink(missing_ok=True) + for file in self.__img_all(game.app_name): + file.unlink(missing_ok=True) if not self.__img_dir(game.app_name).is_dir(): self.__img_dir(game.app_name).mkdir() # Load image checksums if not self.__img_json(game.app_name).is_file(): - json_data: Dict = dict(zip(self.__img_tall_types, [None] * len(self.__img_tall_types))) + json_data: Dict = dict(zip(self.__img_types, [None] * len(self.__img_types))) else: json_data = json.load(open(self.__img_json(game.app_name), "r")) + # Only download the best matching candidate for each image category + def best_match(key_images: List, image_types: Tuple) -> Dict: + matches = sorted( + filter(lambda image: image["type"] in image_types, key_images), + key=lambda x: image_types.index(x["type"]) if x["type"] in image_types else len(image_types), + reverse=False, + ) + try: + best = matches[0] + except IndexError as e: + best = {} + return best + + candidates = tuple( + image + for image in [ + best_match(game.metadata.get("keyImages", []), self.__img_tall_types), + best_match(game.metadata.get("keyImages", []), self.__img_wide_types), + best_match(game.metadata.get("keyImages", []), self.__img_logo_types), + ] + if bool(image) + ) + # lk: Find updates or initialize if images are missing. # lk: `updates` will be empty for games without images # lk: so everything below it is skipped updates = [] - if not ( - self.__img_desktop_icon(game.app_name).is_file() - and self.__img_tall_color(game.app_name).is_file() - and self.__img_tall_gray(game.app_name).is_file() - ): + if not all(file.is_file() for file in self.__img_all(game.app_name)): # lk: fast path for games without images, convert Rare's logo - if not game.metadata.get("keyImages", []): - cache_data: Dict = dict(zip(self.__img_tall_types, [None] * len(self.__img_tall_types))) - cache_data["DieselGameBoxTall"] = open( - resources_path.joinpath("images", "cover.png"), "rb" - ).read() + if not candidates: + cache_data: Dict = dict(zip(self.__img_types, [None] * len(self.__img_types))) + with open(resources_path.joinpath("images", "cover.png"), "rb") as fd: + cache_data["DieselGameBoxTall"] = fd.read() + fd.seek(0) + cache_data["DieselGameBoxWide"] = fd.read() # cache_data["DieselGameBoxLogo"] = open( # resources_path.joinpath("images", "Rare_nonsquared.png"), "rb").read() self.__convert(game, cache_data) json_data["cache"] = None json_data["scale"] = ImageSize.Image.pixel_ratio - json_data["size"] = ImageSize.Image.size.__str__() + json_data["size"] = {"w": ImageSize.Image.size.width(), "h": ImageSize.Image.size.height()} json.dump(json_data, open(self.__img_json(game.app_name), "w")) else: - updates = [image for image in game.metadata["keyImages"] if image["type"] in self.__img_tall_types] + updates = [image for image in candidates if image["type"] in self.__img_types] else: - for image in game.metadata.get("keyImages", []): - if image["type"] in self.__img_tall_types: + for image in candidates: + if image["type"] in self.__img_types: if image["type"] not in json_data.keys() or json_data[image["type"]] != image["md5"]: updates.append(image) return updates, json_data - def __download(self, updates, json_data, game, use_async: bool = False) -> bool: + def __download(self, updates: List, json_data: Dict, game: Game, use_async: bool = False) -> bool: # Decompress existing image.cache if not self.__img_cache(game.app_name).is_file(): - cache_data = dict(zip(self.__img_tall_types, [None] * len(self.__img_tall_types))) + cache_data = dict(zip(self.__img_types, [None] * len(self.__img_types))) else: cache_data = self.__decompress(game) # lk: filter updates again against the cache now that it is available - updates = [ + # images in cache don't need to be downloaded again. + downloads = [ image for image in updates - if cache_data.get(image["type"], None) is None or json_data[image["type"]] != image["md5"] + if (cache_data.get(image["type"], None) is None or json_data[image["type"]] != image["md5"]) ] # Download # # lk: Keep this here, so I don't have to go looking for it again, # # lk: it might be useful in the future. - # if use_async and len(updates) > 1: + # if use_async: # session = FuturesSession(max_workers=len(self.__img_types)) # image_requests = [] - # for image in updates: + # for image in downloads: # logger.info(f"Downloading {image['type']} for {game.app_title}") # json_data[image["type"]] = image["md5"] - # payload = {"resize": 1, "w": ImageSize.Image.size.width(), "h": ImageSize.Image.size.height()} + # if image["type"] in self.__img_tall_types: + # payload = {"resize": 1, "w": ImageSize.Image.size.width(), "h": ImageSize.Image.size.height()} + # elif image["type"] in self.__img_wide_types: + # payload = {"resize": 1, "w": ImageSize.ImageWide.size.width(), "h": ImageSize.ImageWide.size.height()} + # else: + # # Set the larger of the sizes for everything else + # payload = {"resize": 1, "w": ImageSize.ImageWide.size.width(), "h": ImageSize.ImageWide.size.height()} # req = session.get(image["url"], params=payload) # req.image_type = image["type"] # image_requests.append(req) # for req in futures.as_completed(image_requests): # cache_data[req.image_type] = req.result().content # else: - for image in updates: + for image in downloads: logger.info(f"Downloading {image['type']} for {game.app_name} ({game.app_title})") json_data[image["type"]] = image["md5"] - payload = {"resize": 1, "w": ImageSize.Image.size.width(), "h": ImageSize.Image.size.height()} + if image["type"] in self.__img_tall_types: + payload = {"resize": 1, "w": ImageSize.Image.size.width(), "h": ImageSize.Image.size.height()} + elif image["type"] in self.__img_wide_types: + payload = {"resize": 1, "w": ImageSize.ImageWide.size.width(), "h": ImageSize.ImageWide.size.height()} + else: + # Set the larger of the sizes for everything else + payload = {"resize": 1, "w": ImageSize.ImageWide.size.width(), "h": ImageSize.ImageWide.size.height()} try: # cache_data[image["type"]] = requests.get(image["url"], params=payload).content cache_data[image["type"]] = requests.get(image["url"], params=payload, timeout=10).content @@ -208,14 +240,13 @@ class ImageManager(QObject): except FileNotFoundError: archive_hash = None - _json_data = json_data.copy() - _json_data["cache"] = archive_hash - _json_data["scale"] = ImageSize.Image.pixel_ratio - _json_data["size"] = {"w": ImageSize.Image.size.width(), "h": ImageSize.Image.size.height()} + json_data["cache"] = archive_hash + json_data["scale"] = ImageSize.Image.pixel_ratio + json_data["size"] = {"w": ImageSize.Image.size.width(), "h": ImageSize.Image.size.height()} # write image.json with open(self.__img_json(game.app_name), "w") as file: - json.dump(_json_data, file) + json.dump(json_data, file) return bool(updates) @@ -226,13 +257,13 @@ class ImageManager(QObject): if ImageManager.__icon_overlay is not None: return ImageManager.__icon_overlay rounded_path = QPainterPath() - margin = 0.1 + margin = 0.05 rounded_path.addRoundedRect( QRectF( rect.width() * margin, rect.height() * margin, rect.width() - (rect.width() * margin * 2), - rect.height() - (rect.width() * margin * 2) + rect.height() - (rect.width() * margin * 2), ), rect.height() * 0.2, rect.height() * 0.2, @@ -251,7 +282,7 @@ class ImageManager(QObject): painter.fillRect(icon.rect(), Qt.transparent) overlay = ImageManager.__generate_icon_overlay(icon.rect()) brush = QBrush(cover) - scale = max(icon.width()/cover.width(), icon.height()/cover.height()) + scale = max(icon.width() / cover.width(), icon.height() / cover.height()) transform = QTransform().scale(scale, scale) brush.setTransform(transform) painter.fillPath(overlay, brush) @@ -262,52 +293,65 @@ class ImageManager(QObject): return icon def __convert(self, game, images, force=False) -> None: - for image in [self.__img_tall_color(game.app_name), self.__img_tall_gray(game.app_name)]: - if force and image.exists(): - image.unlink(missing_ok=True) + for file in self.__img_all(game.app_name): + if force and file.exists(): + file.unlink(missing_ok=True) - cover_data = None - for image_type in self.__img_tall_types: - if images[image_type] is not None: - cover_data = images[image_type] - break + def find_image_data(image_types: Tuple): + data = None + for image_type in image_types: + if images.get(image_type, None) is not None: + data = images[image_type] + break + return data - cover = QImage() - cover.loadFromData(cover_data) - cover.convertToFormat(QImage.Format_ARGB32_Premultiplied) - # lk: Images are not always 4/3, crop them to size - factor = min(cover.width() // 3, cover.height() // 4) - rem_w = (cover.width() - factor * 3) // 2 - rem_h = (cover.height() - factor * 4) // 2 - cover = cover.copy(rem_w, rem_h, factor * 3, factor * 4) + tall_data = find_image_data(self.__img_tall_types) + wide_data = find_image_data(self.__img_wide_types) + logo_data = find_image_data(self.__img_logo_types) - if images["DieselGameBoxLogo"] is not None: - logo = QImage() - logo.loadFromData(images["DieselGameBoxLogo"]) - logo.convertToFormat(QImage.Format_ARGB32_Premultiplied) - if logo.width() > cover.width(): - logo = logo.scaled(cover.width(), cover.height(), Qt.KeepAspectRatio, Qt.SmoothTransformation) - painter = QPainter(cover) - painter.drawImage((cover.width() - logo.width()) // 2, cover.height() - logo.height(), logo) - painter.end() + def convert_image(image_data, logo_data, preset: ImageSize.Preset) -> QImage: + image = QImage() + image.loadFromData(image_data) + image.convertToFormat(QImage.Format_ARGB32_Premultiplied) + # lk: Images are not always at the correct aspect ratio, so crop them to size + wr, hr = preset.aspect_ratio + factor = min(image.width() // wr, image.height() // hr) + rem_w = (image.width() - factor * wr) // 2 + rem_h = (image.height() - factor * hr) // 2 + image = image.copy(rem_w, rem_h, factor * wr, factor * hr) - cover = cover.scaled(ImageSize.Image.size, Qt.KeepAspectRatio, Qt.SmoothTransformation) + if logo_data is not None: + logo = QImage() + logo.loadFromData(logo_data) + logo.convertToFormat(QImage.Format_ARGB32_Premultiplied) + if logo.width() > image.width(): + logo = logo.scaled(image.width(), image.height(), Qt.KeepAspectRatio, Qt.SmoothTransformation) + painter = QPainter(image) + painter.drawImage((image.width() - logo.width()) // 2, image.height() - logo.height(), logo) + painter.end() - icon = self.__convert_icon(cover) - icon.save(str(self.__img_desktop_icon(game.app_name)), format=desktop_icon_suffix().upper()) + return image.scaled(preset.size, Qt.KeepAspectRatio, Qt.SmoothTransformation) - # this is not required if we ever want to re-apply the alpha channel - # cover = cover.convertToFormat(QImage.Format_Indexed8) + tall = convert_image(tall_data, logo_data, ImageSize.Image) + wide = convert_image(wide_data, logo_data, ImageSize.ImageWide) - # add the alpha channel back to the cover - cover = cover.convertToFormat(QImage.Format_ARGB32_Premultiplied) + icon = self.__convert_icon(tall) + icon.save(str(self.__img_icon(game.app_name)), format=desktop_icon_suffix().upper()) - cover.save(str(self.__img_tall_color(game.app_name)), format="PNG") - # quick way to convert to grayscale - cover = cover.convertToFormat(QImage.Format_Grayscale8) - # add the alpha channel back to the grayscale cover - cover = cover.convertToFormat(QImage.Format_ARGB32_Premultiplied) - cover.save(str(self.__img_tall_gray(game.app_name)), format="PNG") + def save_image(image: QImage, color_path: Path, gray_path: Path): + # this is not required if we ever want to re-apply the alpha channel + # image = image.convertToFormat(QImage.Format_Indexed8) + # add the alpha channel back to the cover + image = image.convertToFormat(QImage.Format_ARGB32_Premultiplied) + image.save(color_path.as_posix(), format="PNG") + # quick way to convert to grayscale + image = image.convertToFormat(QImage.Format_Grayscale8) + # add the alpha channel back to the grayscale cover + image = image.convertToFormat(QImage.Format_ARGB32_Premultiplied) + image.save(gray_path.as_posix(), format="PNG") + + save_image(tall, self.__img_tall_color(game.app_name), self.__img_tall_gray(game.app_name)) + save_image(wide, self.__img_wide_color(game.app_name), self.__img_wide_gray(game.app_name)) def __compress(self, game: Game, data: Dict) -> None: archive = open(self.__img_cache(game.app_name), "wb") @@ -321,27 +365,42 @@ class ImageManager(QObject): data = zlib.decompress(archive.read()) data = pickle.loads(data) except zlib.error: - data = dict(zip(self.__img_tall_types, [None] * len(self.__img_tall_types))) + data = dict(zip(self.__img_types, [None] * len(self.__img_types))) finally: archive.close() return data + def __append_to_queue(self, game: Game): + self.__worker_app_names.add(game.app_name) + + @pyqtSlot(object) + def __remove_from_queue(self, game: Game): + self.__worker_app_names.remove(game.app_name) + def download_image( - self, game: Game, load_callback: Callable[[], None], priority: int, force: bool = False + self, game: Game, load_callback: Callable[[], None], priority: int, force: bool = False ) -> None: if game.app_name in self.__worker_app_names: return - self.__worker_app_names.add(game.app_name) + self.__append_to_queue(game) updates, json_data = self.__prepare_download(game, force) if not updates: - self.__worker_app_names.remove(game.app_name) + self.__remove_from_queue(game) load_callback() else: - image_worker = ImageManager.Worker(self.__download, updates, json_data, game) - image_worker.signals.completed.connect(lambda g: self.__worker_app_names.remove(g.app_name)) + # Copy the data because we are going to be a thread and we modify them later on + image_worker = ImageManager.Worker(self.__download, updates.copy(), json_data.copy(), game) + image_worker.signals.completed.connect(self.__remove_from_queue) image_worker.signals.completed.connect(load_callback) self.threadpool.start(image_worker, priority) + def download_image_launch( + self, game: Game, callback: Callable[[], None], priority: int, force: bool = False + ) -> None: + if self.__img_cache(game.app_name).is_file() and not force: + return + self.download_image(game, callback, priority, force) + def download_image_blocking(self, game: Game, force: bool = False) -> None: updates, json_data = self.__prepare_download(game, force) if not updates: @@ -350,17 +409,17 @@ class ImageManager(QObject): self.__download(updates, json_data, game, use_async=True) def __get_cover( - self, container: Union[Type[QPixmap], Type[QImage]], app_name: str, color: bool = True + self, container: Union[Type[QPixmap], Type[QImage]], app_name: str, color: bool ) -> Union[QPixmap, QImage]: ret = container() if not app_name: raise RuntimeError("app_name is an empty string") if color: if self.__img_tall_color(app_name).is_file(): - ret.load(str(self.__img_tall_color(app_name))) + ret.load(self.__img_tall_color(app_name).as_posix()) else: if self.__img_tall_gray(app_name).is_file(): - ret.load(str(self.__img_tall_gray(app_name))) + ret.load(self.__img_tall_gray(app_name).as_posix()) if not ret.isNull(): ret.setDevicePixelRatio(ImageSize.Image.pixel_ratio) # lk: Scaling happens at painting. It might be inefficient so leave this here as an alternative