QtRequests: Handle multiple requests concurrently and use caching
It is still junky but less so. It allows us to use QNetworkRequest's caching mechanism instead of doing so on our own per-case.
This commit is contained in:
loathingKernel
parent
6c0a92966e
commit
0ef2497afb
|
|
@ -86,9 +86,6 @@ class MainTabWidget(QTabWidget):
|
|||
if index == self.games_index:
|
||||
self.games_tab.setCurrentWidget(self.games_tab.games_page)
|
||||
|
||||
if not self.args.offline and index == self.store_index:
|
||||
self.store_tab.load()
|
||||
|
||||
def resizeEvent(self, event):
|
||||
self.tab_bar.setMinimumWidth(self.width())
|
||||
super(MainTabWidget, self).resizeEvent(event)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from PyQt5.QtWidgets import QWidget
|
|||
|
||||
from rare import __version__, __codename__
|
||||
from rare.ui.components.tabs.settings.about import Ui_About
|
||||
from rare.utils.qt_requests import QtRequestManager
|
||||
from rare.utils.qt_requests import QtRequests
|
||||
|
||||
logger = getLogger("About")
|
||||
|
||||
|
|
@ -34,7 +34,7 @@ class About(QWidget):
|
|||
self.ui.open_browser.setVisible(False)
|
||||
self.ui.open_browser.setEnabled(False)
|
||||
|
||||
self.manager = QtRequestManager("json")
|
||||
self.manager = QtRequests(parent=self)
|
||||
self.manager.get(
|
||||
"https://api.github.com/repos/RareDevs/Rare/releases/latest",
|
||||
self.update_available_finished,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from PyQt5.QtGui import QShowEvent, QHideEvent
|
||||
from PyQt5.QtWidgets import QStackedWidget, QTabWidget
|
||||
from legendary.core import LegendaryCore
|
||||
|
||||
|
|
@ -26,7 +27,7 @@ class Shop(QStackedWidget):
|
|||
self.shop = ShopWidget(cache_dir(), self.core, self.api_core)
|
||||
self.wishlist_widget = Wishlist(self.api_core)
|
||||
|
||||
self.store_tabs = QTabWidget()
|
||||
self.store_tabs = QTabWidget(parent=self)
|
||||
self.store_tabs.addTab(self.shop, self.tr("Games"))
|
||||
self.store_tabs.addTab(self.wishlist_widget, self.tr("Wishlist"))
|
||||
|
||||
|
|
@ -50,15 +51,23 @@ class Shop(QStackedWidget):
|
|||
self.api_core.update_wishlist.connect(self.update_wishlist)
|
||||
self.wishlist_widget.update_wishlist_signal.connect(self.update_wishlist)
|
||||
|
||||
def showEvent(self, a0: QShowEvent) -> None:
|
||||
if a0.spontaneous() or self.init:
|
||||
return super().showEvent(a0)
|
||||
self.shop.load()
|
||||
self.wishlist_widget.update_wishlist()
|
||||
self.init = True
|
||||
return super().showEvent(a0)
|
||||
|
||||
def hideEvent(self, a0: QHideEvent) -> None:
|
||||
if a0.spontaneous():
|
||||
return super().hideEvent(a0)
|
||||
# TODO: Implement store unloading
|
||||
return super().hideEvent(a0)
|
||||
|
||||
def update_wishlist(self):
|
||||
self.shop.update_wishlist()
|
||||
|
||||
def load(self):
|
||||
if not self.init:
|
||||
self.init = True
|
||||
self.shop.load()
|
||||
self.wishlist_widget.update_wishlist()
|
||||
|
||||
def show_game_info(self, data):
|
||||
self.info.update_game(data)
|
||||
self.setCurrentIndex(2)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ from rare.components.tabs.shop.constants import (
|
|||
remove_from_wishlist_query,
|
||||
)
|
||||
from rare.components.tabs.shop.shop_models import BrowseModel
|
||||
from rare.utils.qt_requests import QtRequestManager
|
||||
from rare.utils.qt_requests import QtRequests
|
||||
from rare.utils.paths import cache_dir
|
||||
|
||||
logger = getLogger("ShopAPICore")
|
||||
graphql_url = "https://www.epicgames.com/graphql"
|
||||
|
|
@ -25,8 +26,8 @@ class ShopApiCore(QObject):
|
|||
self.language_code: str = lc
|
||||
self.country_code: str = cc
|
||||
self.locale = f"{self.language_code}-{self.country_code}"
|
||||
self.manager = QtRequestManager()
|
||||
self.auth_manager = QtRequestManager(authorization_token=auth_token)
|
||||
self.manager = QtRequests(parent=self)
|
||||
self.auth_manager = QtRequests(token=auth_token, parent=self)
|
||||
|
||||
self.browse_active = False
|
||||
self.next_browse_request = tuple(())
|
||||
|
|
@ -52,6 +53,7 @@ class ShopApiCore(QObject):
|
|||
def get_wishlist(self, handle_func):
|
||||
self.auth_manager.post(
|
||||
graphql_url,
|
||||
lambda data: self._handle_wishlist(data, handle_func),
|
||||
{
|
||||
"query": wishlist_query,
|
||||
"variables": {
|
||||
|
|
@ -59,7 +61,6 @@ class ShopApiCore(QObject):
|
|||
"locale": f"{self.language_code}-{self.country_code}",
|
||||
},
|
||||
},
|
||||
lambda data: self._handle_wishlist(data, handle_func),
|
||||
)
|
||||
|
||||
def _handle_wishlist(self, data, handle_func):
|
||||
|
|
@ -95,7 +96,7 @@ class ShopApiCore(QObject):
|
|||
}
|
||||
|
||||
self.manager.post(
|
||||
graphql_url, payload, lambda data: self._handle_search(data, handle_func)
|
||||
graphql_url, lambda data: self._handle_search(data, handle_func), payload,
|
||||
)
|
||||
|
||||
def _handle_search(self, data, handle_func):
|
||||
|
|
@ -189,8 +190,8 @@ class ShopApiCore(QObject):
|
|||
}
|
||||
self.auth_manager.post(
|
||||
graphql_url,
|
||||
payload,
|
||||
lambda data: self._handle_add_to_wishlist(data, handle_func),
|
||||
payload,
|
||||
)
|
||||
|
||||
def _handle_add_to_wishlist(self, data, handle_func):
|
||||
|
|
@ -216,8 +217,8 @@ class ShopApiCore(QObject):
|
|||
}
|
||||
self.auth_manager.post(
|
||||
graphql_url,
|
||||
payload,
|
||||
lambda data: self._handle_remove_from_wishlist(data, handle_func),
|
||||
payload,
|
||||
)
|
||||
|
||||
def _handle_remove_from_wishlist(self, data, handle_func):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import os
|
||||
from logging import getLogger
|
||||
|
||||
from PyQt5.QtCore import Qt, pyqtSignal
|
||||
|
|
@ -14,8 +13,8 @@ from PyQt5.QtWidgets import (
|
|||
)
|
||||
|
||||
from rare.utils.misc import icon as qta_icon
|
||||
from rare.utils.paths import tmp_dir
|
||||
from rare.utils.qt_requests import QtRequestManager
|
||||
from rare.utils.paths import cache_dir
|
||||
from rare.utils.qt_requests import QtRequests
|
||||
|
||||
logger = getLogger("ExtraWidgets")
|
||||
|
||||
|
|
@ -81,8 +80,10 @@ class ImageLabel(QLabel):
|
|||
|
||||
def __init__(self, parent=None):
|
||||
super(ImageLabel, self).__init__(parent=parent)
|
||||
self.path = tmp_dir()
|
||||
self.manager = QtRequestManager("bytes")
|
||||
self.manager = QtRequests(
|
||||
cache=str(cache_dir().joinpath("store")),
|
||||
parent=self
|
||||
)
|
||||
|
||||
def update_image(self, url, name="", size: tuple = (240, 320)):
|
||||
self.setFixedSize(*size)
|
||||
|
|
@ -95,11 +96,7 @@ class ImageLabel(QLabel):
|
|||
else:
|
||||
name_extension = "tall"
|
||||
self.name = f"{self.name}_{name_extension}.png"
|
||||
if not os.path.exists(os.path.join(self.path, self.name)):
|
||||
self.manager.get(url, self.image_ready)
|
||||
# self.request.finished.connect(self.image_ready)
|
||||
else:
|
||||
self.show_image()
|
||||
self.manager.get(url, self.image_ready)
|
||||
|
||||
def image_ready(self, data):
|
||||
try:
|
||||
|
|
@ -115,17 +112,9 @@ class ImageLabel(QLabel):
|
|||
transformMode=Qt.SmoothTransformation,
|
||||
)
|
||||
|
||||
image.save(os.path.join(self.path, self.name))
|
||||
|
||||
pixmap = QPixmap().fromImage(image)
|
||||
self.setPixmap(pixmap)
|
||||
|
||||
def show_image(self):
|
||||
self.image = QPixmap(os.path.join(self.path, self.name)).scaled(
|
||||
*self.img_size, transformMode=Qt.SmoothTransformation
|
||||
)
|
||||
self.setPixmap(self.image)
|
||||
|
||||
|
||||
class ButtonLineEdit(QLineEdit):
|
||||
buttonClicked = pyqtSignal()
|
||||
|
|
|
|||
|
|
@ -1,109 +1,139 @@
|
|||
import json
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from email.message import Message
|
||||
from logging import getLogger
|
||||
from typing import Callable
|
||||
from typing import Callable, Dict, TypeVar, List, Tuple
|
||||
from typing import Union
|
||||
|
||||
from PyQt5.QtCore import QObject, pyqtSignal, QUrl, QJsonParseError, QJsonDocument
|
||||
from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest, QNetworkReply
|
||||
import orjson
|
||||
from PyQt5.QtCore import QObject, pyqtSignal, QUrl, QUrlQuery, pyqtSlot
|
||||
from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest, QNetworkReply, QNetworkDiskCache
|
||||
|
||||
logger = getLogger("QtRequests")
|
||||
|
||||
|
||||
class QtRequestManager(QObject):
|
||||
data_ready = pyqtSignal(object)
|
||||
request = None
|
||||
request_active = None
|
||||
|
||||
def __init__(self, type: str = "json", authorization_token: str = None):
|
||||
super(QtRequestManager, self).__init__()
|
||||
self.manager = QNetworkAccessManager()
|
||||
self.type = type
|
||||
self.authorization_token = authorization_token
|
||||
self.request_queue = []
|
||||
|
||||
def post(self, url: str, payload: dict, handle_func):
|
||||
if not self.request_active:
|
||||
request = QNetworkRequest(QUrl(url))
|
||||
request.setHeader(QNetworkRequest.ContentTypeHeader, "application/json")
|
||||
self.request_active = RequestQueueItem(handle_func=handle_func)
|
||||
payload = json.dumps(payload).encode("utf-8")
|
||||
|
||||
request.setHeader(
|
||||
QNetworkRequest.UserAgentHeader,
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36",
|
||||
)
|
||||
|
||||
if self.authorization_token is not None:
|
||||
request.setRawHeader(
|
||||
b"Authorization", self.authorization_token.encode()
|
||||
)
|
||||
|
||||
self.request = self.manager.post(request, payload)
|
||||
self.request.finished.connect(self.prepare_data)
|
||||
|
||||
else:
|
||||
self.request_queue.append(
|
||||
RequestQueueItem(
|
||||
method="post", url=url, payload=payload, handle_func=handle_func
|
||||
)
|
||||
)
|
||||
|
||||
def get(self, url: str, handle_func: Callable[[Union[dict, bytes]], None]):
|
||||
if not self.request_active:
|
||||
request = QNetworkRequest(QUrl(url))
|
||||
request.setHeader(
|
||||
QNetworkRequest.UserAgentHeader,
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36",
|
||||
)
|
||||
|
||||
self.request_active = RequestQueueItem(handle_func=handle_func)
|
||||
self.request = self.manager.get(request)
|
||||
self.request.finished.connect(self.prepare_data)
|
||||
else:
|
||||
self.request_queue.append(
|
||||
RequestQueueItem(method="get", url=url, handle_func=handle_func)
|
||||
)
|
||||
|
||||
def prepare_data(self):
|
||||
# self.request_active = False
|
||||
data = {} if self.type == "json" else b""
|
||||
if self.request:
|
||||
try:
|
||||
if self.request.error() == QNetworkReply.NoError:
|
||||
if self.type == "json":
|
||||
error = QJsonParseError()
|
||||
json_data = QJsonDocument.fromJson(
|
||||
self.request.readAll().data(), error
|
||||
)
|
||||
if QJsonParseError.NoError == error.error:
|
||||
data = json.loads(json_data.toJson().data().decode())
|
||||
else:
|
||||
logger.error(error.errorString())
|
||||
else:
|
||||
data = self.request.readAll().data()
|
||||
|
||||
except RuntimeError as e:
|
||||
logger.error(str(e))
|
||||
self.request_active.handle_func(data)
|
||||
self.request.deleteLater()
|
||||
self.request_active = None
|
||||
|
||||
if self.request_queue:
|
||||
if self.request_queue[0].method == "post":
|
||||
self.post(
|
||||
self.request_queue[0].url,
|
||||
self.request_queue[0].payload,
|
||||
self.request_queue[0].handle_func,
|
||||
)
|
||||
else:
|
||||
self.get(self.request_queue[0].url, self.request_queue[0].handle_func)
|
||||
self.request_queue.pop(0)
|
||||
REQUEST_LIMIT = 8
|
||||
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
|
||||
RequestHandler = TypeVar("RequestHandler", bound=Callable[[Union[Dict, bytes]], None])
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestQueueItem:
|
||||
method: str = None
|
||||
url: str = None
|
||||
handle_func: Callable[[Union[dict, bytes]], None] = None
|
||||
payload: dict = None
|
||||
url: QUrl = None
|
||||
payload: Dict = field(default_factory=dict)
|
||||
params: Dict = field(default_factory=dict)
|
||||
handlers: List[RequestHandler] = field(default_factory=list)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.method == other.method and self.url == other.url
|
||||
|
||||
|
||||
class QtRequests(QObject):
|
||||
data_ready = pyqtSignal(object)
|
||||
|
||||
def __init__(self, cache: str = None, token: str = None, parent=None):
|
||||
super(QtRequests, self).__init__(parent=parent)
|
||||
self.log = getLogger(f"{type(self).__name__}_{type(parent).__name__}")
|
||||
self.manager = QNetworkAccessManager(self)
|
||||
self.manager.finished.connect(self.__on_finished)
|
||||
self.manager.finished.connect(self.__process_next)
|
||||
self.cache = None
|
||||
if cache is not None:
|
||||
self.log.debug("Using cache dir %s", cache)
|
||||
self.cache = QNetworkDiskCache(self)
|
||||
self.cache.setCacheDirectory(cache)
|
||||
self.manager.setCache(self.cache)
|
||||
if token is not None:
|
||||
self.log.debug("Manager is authorized")
|
||||
self.token = token
|
||||
|
||||
self.__pending_requests = []
|
||||
self.__active_requests = {}
|
||||
|
||||
@staticmethod
|
||||
def __prepare_query(url, params) -> QUrl:
|
||||
url = QUrl(url)
|
||||
query = QUrlQuery(url)
|
||||
for k, v in params.items():
|
||||
query.addQueryItem(str(k), str(v))
|
||||
url.setQuery(query)
|
||||
return url
|
||||
|
||||
def __prepare_request(self, item: RequestQueueItem) -> QNetworkRequest:
|
||||
request = QNetworkRequest(item.url)
|
||||
request.setHeader(QNetworkRequest.ContentTypeHeader, "application/json;charset=UTF-8")
|
||||
request.setHeader(QNetworkRequest.UserAgentHeader, USER_AGENT)
|
||||
request.setAttribute(QNetworkRequest.RedirectPolicyAttribute, QNetworkRequest.NoLessSafeRedirectPolicy)
|
||||
if self.cache is not None:
|
||||
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache)
|
||||
if self.token is not None:
|
||||
request.setRawHeader(b"Authorization", self.token.encode())
|
||||
return request
|
||||
|
||||
def __post(self, item: RequestQueueItem):
|
||||
request = self.__prepare_request(item)
|
||||
payload = orjson.dumps(item.payload) # pylint: disable=maybe-no-member
|
||||
reply = self.manager.post(request, payload)
|
||||
reply.errorOccurred.connect(self.__on_error)
|
||||
self.__active_requests[reply] = item
|
||||
|
||||
def post(self, url: str, handler: RequestHandler, payload: dict):
|
||||
item = RequestQueueItem(method="post", url=QUrl(url), payload=payload, handlers=[handler])
|
||||
if len(self.__active_requests) < REQUEST_LIMIT:
|
||||
self.__post(item)
|
||||
else:
|
||||
self.__pending_requests.append(item)
|
||||
|
||||
def __get(self, item: RequestQueueItem):
|
||||
request = self.__prepare_request(item)
|
||||
reply = self.manager.get(request)
|
||||
reply.errorOccurred.connect(self.__on_error)
|
||||
self.__active_requests[reply] = item
|
||||
|
||||
def get(self, url: str, handler: RequestHandler, payload: Dict = None, params: Dict = None):
|
||||
url = self.__prepare_query(url, params) if params is not None else QUrl(url)
|
||||
item = RequestQueueItem(method="get", url=url, payload=payload, handlers=[handler])
|
||||
if len(self.__active_requests) < REQUEST_LIMIT:
|
||||
self.__get(item)
|
||||
else:
|
||||
self.__pending_requests.append(item)
|
||||
|
||||
def __on_error(self, error: QNetworkReply.NetworkError) -> None:
|
||||
self.log.error(error)
|
||||
|
||||
@staticmethod
|
||||
def __parse_content_type(header) -> Tuple[str, str]:
|
||||
# lk: this looks weird but `cgi` is deprecated, PEP 594 suggests this way of parsing MIME
|
||||
m = Message()
|
||||
m['content-type'] = header
|
||||
return m.get_content_type(), m.get_content_charset()
|
||||
|
||||
def __process_next(self):
|
||||
if self.__pending_requests:
|
||||
item = self.__pending_requests.pop(0)
|
||||
if item.method == "post":
|
||||
self.__post(item)
|
||||
elif item.method == "get":
|
||||
self.__get(item)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
@pyqtSlot(QNetworkReply)
|
||||
def __on_finished(self, reply: QNetworkReply):
|
||||
item = self.__active_requests.pop(reply, None)
|
||||
if item is None:
|
||||
self.log.error("QNetworkReply: %s without associated item", reply.url().toString())
|
||||
reply.deleteLater()
|
||||
return
|
||||
if reply.error():
|
||||
self.log.error(reply.errorString())
|
||||
else:
|
||||
mimetype, charset = self.__parse_content_type(reply.header(QNetworkRequest.ContentTypeHeader))
|
||||
maintype, subtype = mimetype.split("/")
|
||||
bin_data = reply.readAll().data()
|
||||
if mimetype == "application/json":
|
||||
data = orjson.loads(bin_data)
|
||||
elif maintype == "image":
|
||||
data = bin_data
|
||||
else:
|
||||
data = None
|
||||
for handler in item.handlers:
|
||||
handler(data)
|
||||
reply.deleteLater()
|
||||
|
|
|
|||
Loading…
Reference in a new issue