CodeMirror/Rare
1
0
Fork 0
mirror of synced 2024-06-02 18:54:41 +12:00
Code Issues Projects Releases Wiki Activity

Update DownloadThread

Browse source
This commit is contained in:
loathingKernel 2022-08-03 02:33:50 +03:00
parent 38dfdc8bc2
commit 3ee789a695
10 changed files with 225 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
rare/app.py
View file

@ -154,7 +154,14 @@ class App(RareApp):
legendary_utils.uninstall_game(self.core, igame.app_name, keep_files=True)
logger.info(f"Uninstalled {igame.title}, because no game files exist")
continue
if not os.path.exists(os.path.join(igame.install_path, igame.executable.replace("\\", "/").lstrip("/"))):
# lk: games that don't have an override and can't find their executable due to case sensitivity
# lk: will still erroneously require verification. This might need to be removed completely
# lk: or be decoupled from the verification requirement
if override_exe := self.core.lgd.config.get(igame.app_name, "override_exe", fallback=""):
igame_executable = override_exe
else:
igame_executable = igame.executable
if not os.path.exists(os.path.join(igame.install_path, igame_executable.replace("\\", "/").lstrip("/"))):
igame.needs_verification = True
self.core.lgd.set_installed_game(igame.app_name, igame)
logger.info(f"{igame.title} needs verification")

24
rare/components/tabs/downloads/__init__.py
View file

@ -2,7 +2,7 @@ import datetime
from logging import getLogger
from typing import List, Dict
from PyQt5.QtCore import QThread, pyqtSignal, QSettings
from PyQt5.QtCore import QThread, pyqtSignal, QSettings, pyqtSlot
from PyQt5.QtWidgets import (
QWidget,
QMessageBox,
@ -21,7 +21,7 @@ from rare.lgndr.downloading import UIUpdate
from rare.models.install import InstallOptionsModel, InstallQueueItemModel
from rare.shared import LegendaryCoreSingleton, GlobalSignalsSingleton
from rare.ui.components.tabs.downloads.downloads_tab import Ui_DownloadsTab
from rare.utils.misc import get_size
from rare.utils.misc import get_size, create_desktop_link
logger = getLogger("Download")
@ -137,7 +137,7 @@ class DownloadsTab(QWidget, Ui_DownloadsTab):
self.queue_widget.update_queue(self.dl_queue)
self.active_game = queue_item.download.game
self.thread = DownloadThread(self.core, queue_item)
self.thread.status.connect(self.status)
self.thread.exit_status.connect(self.status)
self.thread.statistics.connect(self.statistics)
self.thread.start()
self.kill_button.setDisabled(False)
@ -146,8 +146,16 @@ class DownloadsTab(QWidget, Ui_DownloadsTab):
self.signals.installation_started.emit(self.active_game.app_name)
def status(self, text):
if text == "finish":
@pyqtSlot(DownloadThread.ExitStatus)
def status(self, result: DownloadThread.ExitStatus):
if result.exit_code == DownloadThread.ExitCode.FINISHED:
if result.shortcuts:
if not create_desktop_link(result.app_name, self.core, "desktop"):
# maybe add it to download summary, to show in finished downloads
pass
else:
logger.info("Desktop shortcut written")
self.dl_name.setText(self.tr("Download finished. Reload library"))
logger.info(f"Download finished: {self.active_game.app_title}")
@ -182,10 +190,10 @@ class DownloadsTab(QWidget, Ui_DownloadsTab):
else:
self.queue_widget.update_queue(self.dl_queue)
elif text[:5] == "error":
QMessageBox.warning(self, "warn", f"Download error: {text[6:]}")
elif result.exit_code == DownloadThread.ExitCode.ERROR:
QMessageBox.warning(self, self.tr("Error"), f"Download error: {result.message}")
elif text == "stop":
elif result.exit_code == DownloadThread.ExitCode.STOPPED:
self.reset_infos()
if w := self.update_widgets.get(self.active_game.app_name):
w.update_button.setDisabled(False)

219
rare/components/tabs/downloads/download_thread.py
View file

@ -1,27 +1,40 @@
import os
import platform
import queue
import sys
import time
from dataclasses import dataclass
from enum import IntEnum
from logging import getLogger
from queue import Empty
from typing import List, Optional, Dict
import psutil
from PyQt5.QtCore import QThread, pyqtSignal, QProcess
from legendary.core import LegendaryCore
from legendary.models.downloading import WriterTask
from rare.lgndr.api_monkeys import DLManagerSignals
from rare.lgndr.cli import LegendaryCLI
from rare.shared import GlobalSignalsSingleton
from rare.models.install import InstallQueueItemModel
from rare.utils.misc import create_desktop_link
from rare.lgndr.downloading import UIUpdate
from rare.models.install import InstallQueueItemModel
from rare.shared import GlobalSignalsSingleton
logger = getLogger("DownloadThread")
class DownloadThread(QThread):
status = pyqtSignal(str)
class ExitCode(IntEnum):
ERROR = 1
STOPPED = 2
FINISHED = 3
@dataclass
class ExitStatus:
app_name: str
exit_code: int
message: str = ""
dlcs: Optional[List[Dict]] = None
sync_saves: bool = False
shortcuts: bool = False
exit_status = pyqtSignal(ExitStatus)
statistics = pyqtSignal(UIUpdate)
def __init__(self, core: LegendaryCore, item: InstallQueueItemModel):
@ -29,166 +42,93 @@ class DownloadThread(QThread):
self.signals = GlobalSignalsSingleton()
self.core: LegendaryCore = core
self.item: InstallQueueItemModel = item
self._kill = False
self.dlm_signals: DLManagerSignals = DLManagerSignals()
def run(self):
start_time = time.time()
dl_stopped = False
_exit_status = DownloadThread.ExitStatus(self.item.download.game.app_name, DownloadThread.ExitCode.ERROR)
start_t = time.time()
try:
self.item.download.dlmanager.start()
time.sleep(1)
while self.item.download.dlmanager.is_alive():
if self._kill:
self.status.emit("stop")
logger.info("Download stopping...")
# The code below is a temporary solution.
# It should be removed once legendary supports stopping downloads more gracefully.
self.item.download.dlmanager.running = False
# send conditions to unlock threads if they aren't already
for cond in self.item.download.dlmanager.conditions:
with cond:
cond.notify()
# make sure threads are dead.
for t in self.item.download.dlmanager.threads:
t.join(timeout=5.0)
if t.is_alive():
logger.warning(f"Thread did not terminate! {repr(t)}")
# clean up all the queues, otherwise this process won't terminate properly
for name, q in zip(
(
"Download jobs",
"Writer jobs",
"Download results",
"Writer results",
),
(
self.item.download.dlmanager.dl_worker_queue,
self.item.download.dlmanager.writer_queue,
self.item.download.dlmanager.dl_result_q,
self.item.download.dlmanager.writer_result_q,
),
):
logger.debug(f'Cleaning up queue "{name}"')
try:
while True:
_ = q.get_nowait()
except Empty:
q.close()
q.join_thread()
except AttributeError:
logger.warning(f"Queue {name} did not close")
if self.item.download.dlmanager.writer_queue:
# cancel installation
self.item.download.dlmanager.writer_queue.put_nowait(WriterTask("", kill=True))
# forcibly kill DL workers that are not actually dead yet
for child in self.item.download.dlmanager.children:
if child.exitcode is None:
child.terminate()
if self.item.download.dlmanager.shared_memory:
# close up shared memory
self.item.download.dlmanager.shared_memory.close()
self.item.download.dlmanager.shared_memory.unlink()
self.item.download.dlmanager.shared_memory = None
self.item.download.dlmanager.kill()
# force kill any threads that are somehow still alive
for proc in psutil.process_iter():
# check whether the process name matches
if (
sys.platform in ["linux", "darwin"]
and proc.name() == "DownloadThread"
):
proc.kill()
elif (
sys.platform == "win32"
and proc.name() == "python.exe"
and proc.create_time() >= start_time
):
proc.kill()
logger.info("Download stopped. It can be continued later.")
dl_stopped = True
try:
if not dl_stopped:
self.statistics.emit(self.item.download.dlmanager.status_queue.get(timeout=1))
self.statistics.emit(self.item.download.dlmanager.status_queue.get(timeout=1.0))
except queue.Empty:
pass
if self.dlm_signals.update:
try:
self.item.download.dlmanager.signals_queue.put(self.dlm_signals, block=False, timeout=1.0)
except queue.Full:
pass
time.sleep(self.item.download.dlmanager.update_interval/10)
self.item.download.dlmanager.join()
except Exception as e:
logger.error(
f"Installation failed after {time.time() - start_time:.02f} seconds: {e}"
)
self.status.emit(f"error {e}")
return
else:
if dl_stopped:
return
self.status.emit("dl_finished")
end_t = time.time()
logger.info(f"Download finished in {end_t - start_time}s")
logger.error(f"Installation failed after {end_t - start_t:.02f} seconds.")
logger.warning(f"The following exception occurred while waiting for the downloader to finish: {e!r}.")
_exit_status.exit_code = DownloadThread.ExitCode.ERROR
_exit_status.message = f"{e!r}"
self.exit_status.emit(_exit_status)
return
else:
end_t = time.time()
if self.dlm_signals.kill is True:
logger.info(f"Download stopped after {end_t - start_t:.02f} seconds.")
_exit_status.exit_code = DownloadThread.ExitCode.STOPPED
self.exit_status.emit(_exit_status)
return
logger.info(f"Download finished in {end_t - start_t:.02f} seconds.")
_exit_status.exit_code = DownloadThread.ExitCode.FINISHED
if self.item.options.overlay:
self.signals.overlay_installation_finished.emit()
self.core.finish_overlay_install(self.item.download.igame)
self.status.emit("finish")
self.exit_status.emit(_exit_status)
return
if not self.item.options.no_install:
postinstall = self.core.install_game(self.item.download.igame)
if postinstall:
# LegendaryCLI(self.core)._handle_postinstall(
# postinstall,
# self.item.download.igame,
# False,
# self.item.options.install_preqs,
# )
self._handle_postinstall(postinstall, self.item.download.igame)
dlcs = self.core.get_dlc_for_game(self.item.download.igame.app_name)
if dlcs:
print("The following DLCs are available for this game:")
for dlc in dlcs:
print(
f" - {dlc.app_title} (App name: {dlc.app_name}, version: {dlc.app_version})"
_exit_status.dlcs.append(
{"app_name": dlc.app_name, "app_title": dlc.app_title, "app_version": dlc.app_version}
)
print(
"Manually installing DLCs works the same; just use the DLC app name instead."
)
if self.item.download.game.supports_cloud_saves and not self.item.download.game.is_dlc:
logger.info(
'This game supports cloud saves, syncing is handled by the "sync-saves" command.'
)
logger.info(
f'To download saves for this game run "legendary sync-saves {self.item.download.game.app_name}"'
)
_exit_status.sync_saves = True
LegendaryCLI(self.core).clean_post_install(
self.item.download.game, self.item.download.igame,
self.item.download.repair, self.item.download.repair_file
)
LegendaryCLI(self.core).clean_post_install(
self.item.download.game,
self.item.download.igame,
self.item.download.repair,
self.item.download.repair_file,
)
if not self.item.options.update and self.item.options.create_shortcut:
if not create_desktop_link(self.item.options.app_name, self.core, "desktop"):
# maybe add it to download summary, to show in finished downloads
pass
else:
logger.info("Desktop shortcut written")
if not self.item.options.update and self.item.options.create_shortcut:
_exit_status.shortcuts = True
self.status.emit("finish")
self.exit_status.emit(_exit_status)
def _handle_postinstall(self, postinstall, igame):
logger.info(f"Postinstall info: {postinstall}")
logger.info("This game lists the following prequisites to be installed:")
logger.info(f'- {postinstall["name"]}: {" ".join((postinstall["path"], postinstall["args"]))}')
if platform.system() == "Windows":
if self.item.options.install_preqs:
if not self.item.options.install_preqs:
logger.info('Marking prerequisites as installed...')
self.core.prereq_installed(self.item.download.igame.app_name)
else:
logger.info('Launching prerequisite executable..')
self.core.prereq_installed(igame.app_name)
req_path, req_exec = os.path.split(postinstall["path"])
work_dir = os.path.join(igame.install_path, req_path)
@ -196,15 +136,14 @@ class DownloadThread(QThread):
proc = QProcess()
proc.setProcessChannelMode(QProcess.MergedChannels)
proc.readyReadStandardOutput.connect(
lambda: logger.debug(
str(proc.readAllStandardOutput().data(), "utf-8", "ignore")
))
proc.start(fullpath, postinstall.get("args", []))
lambda: logger.debug(str(proc.readAllStandardOutput().data(), "utf-8", "ignore"))
)
proc.setNativeArguments(postinstall.get("args", []))
proc.setWorkingDirectory(work_dir)
proc.start(fullpath)
proc.waitForFinished() # wait, because it is inside the thread
else:
self.core.prereq_installed(self.item.download.igame.app_name)
else:
logger.info("Automatic installation not available on Linux.")
def kill(self):
self._kill = True
self.dlm_signals.kill = True

22
rare/lgndr/api_monkeys.py
View file

@ -1,7 +1,6 @@
import logging
from dataclasses import dataclass
from PyQt5.QtWidgets import QLabel
from typing_extensions import Protocol
@ -18,6 +17,27 @@ def verify_stdout(a0: int, a1: int, a2: float, a3: float) -> None:
print(f"Verification progress: {a0}/{a1} ({a2:.01f}%) [{a3:.1f} MiB/s]\t\r")
class DLManagerSignals:
_kill = False
_update = False
@property
def kill(self) -> bool:
self._update = False
return self._kill
@kill.setter
def kill(self, value: bool) -> None:
self._update = True
self._kill = value
@property
def update(self) -> bool:
_update = self._update
self._update = False
return _update
@dataclass
class LgndrIndirectStatus:
success: bool = False

42
rare/lgndr/cli.py
View file

@ -1,6 +1,7 @@
import logging
import os
import time
import subprocess
from typing import Optional, Union, Tuple
from legendary.cli import LegendaryCLI as LegendaryCLIReal
@ -15,6 +16,7 @@ from .core import LegendaryCore
from .manager import DLManager
# fmt: off
class LegendaryCLI(LegendaryCLIReal):
# noinspection PyMissingConstructor
@ -205,8 +207,42 @@ class LegendaryCLI(LegendaryCLIReal):
self.core.uninstall_tag(old_igame)
self.core.install_game(old_igame)
def handle_postinstall(self, postinstall, igame, yes=False):
super(LegendaryCLI, self)._handle_postinstall(postinstall, igame, yes)
def _handle_postinstall(self, postinstall, igame, yes=False, choice=False):
# Override logger for the local context to use message as part of the indirect return value
logger = LgndrIndirectLogger(LgndrIndirectStatus(), self.logger)
# noinspection PyShadowingBuiltins
def print(x): self.logger.info(x) if x else None
# noinspection PyShadowingBuiltins
def input(x): return 'y' if choice else 'i'
print('\nThis game lists the following prequisites to be installed:')
print(f'- {postinstall["name"]}: {" ".join((postinstall["path"], postinstall["args"]))}')
print('')
if os.name == 'nt':
if yes:
c = 'n' # we don't want to launch anything, just silent install.
else:
choice = input('Do you wish to install the prerequisites? ([y]es, [n]o, [i]gnore): ')
c = choice.lower()[0]
print('')
if c == 'i': # just set it to installed
logger.info('Marking prerequisites as installed...')
self.core.prereq_installed(igame.app_name)
elif c == 'y': # set to installed and launch installation
logger.info('Launching prerequisite executable..')
self.core.prereq_installed(igame.app_name)
req_path, req_exec = os.path.split(postinstall['path'])
work_dir = os.path.join(igame.install_path, req_path)
fullpath = os.path.join(work_dir, req_exec)
try:
p = subprocess.Popen([fullpath, postinstall['args']], cwd=work_dir, shell=True)
p.wait()
except Exception as e:
logger.error(f'Failed to run prereq executable with: {e!r}')
else:
logger.info('Automatic installation not available on Linux.')
def uninstall_game(self, args: LgndrUninstallGameArgs) -> None:
# Override logger for the local context to use message as part of the indirect return value
@ -489,3 +525,5 @@ class LegendaryCLI(LegendaryCLIReal):
igame.install_path = new_path
self.core.install_game(igame)
logger.info('Finished.')
# fmt: on

13
rare/lgndr/core.py
View file

@ -8,7 +8,11 @@ from legendary.models.manifest import ManifestMeta
from .api_exception import LgndrException, LgndrCoreLogHandler
from .manager import DLManager
# import legendary.core
# legendary.core.DLManager = DLManager
# fmt: off
class LegendaryCore(LegendaryCoreReal):
def __init__(self, override_config=None, timeout=10.0):
@ -48,7 +52,11 @@ class LegendaryCore(LegendaryCoreReal):
)
# lk: monkeypatch run_real (the method that emits the stats) into DLManager
dlm.run_real = DLManager.run_real.__get__(dlm, DLManager)
# lk: set the queue for reporting statistics back the UI
dlm.status_queue = Queue()
# lk: set the queue to send control signals to the DLManager
# lk: this doesn't exist in the original class, but it is monkeypatched in
dlm.signals_queue = Queue()
return dlm, analysis, igame
def uninstall_game(self, installed_game: InstalledGame, delete_files=True, delete_root_directory=False):
@ -79,6 +87,11 @@ class LegendaryCore(LegendaryCoreReal):
dlm, analysis_result, igame = super(LegendaryCore, self).prepare_overlay_install(path)
# lk: monkeypatch status_q (the queue for download stats)
dlm.run_real = DLManager.run_real.__get__(dlm, DLManager)
# lk: set the queue for reporting statistics back the UI
dlm.status_queue = Queue()
# lk: set the queue to send control signals to the DLManager
# lk: this doesn't exist in the original class, but it is monkeypatched in
dlm.signals_queue = Queue()
return dlm, analysis_result, igame
# fmt: on

3
rare/lgndr/downloading.py
View file

@ -5,7 +5,8 @@ from typing import Optional
@dataclass
class UIUpdate:
"""
Status update object sent from the manager to the CLI/GUI to update status indicators
Status update object sent from the manager to the CLI/GUI to update status indicators
Inheritance doesn't work due to optional arguments in UIUpdate proper
"""
progress: float
download_speed: float

42
rare/lgndr/manager.py
View file

@ -1,5 +1,6 @@
import logging
import os
import queue
import time
from multiprocessing import Queue as MPQueue
from multiprocessing.shared_memory import SharedMemory
@ -11,10 +12,14 @@ from legendary.downloader.mp.workers import DLWorker, FileWorker
from legendary.models.downloading import ChunkTask, SharedMemorySegment, TerminateWorkerTask
from .downloading import UIUpdate
from .api_monkeys import DLManagerSignals
# fmt: off
class DLManager(DLManagerReal):
# fmt: off
# Rare: prototype to avoid undefined variable in type checkers
signals_queue: MPQueue
# @staticmethod
def run_real(self):
self.shared_memory = SharedMemory(create=True, size=self.max_shared_memory)
@ -77,6 +82,9 @@ class DLManager(DLManagerReal):
last_update = time.time()
# Rare: kill requested
kill_request = False
while processed_tasks < num_tasks:
delta = time.time() - last_update
if not delta:
@ -121,7 +129,7 @@ class DLManager(DLManagerReal):
rt_hours = rt_minutes = rt_seconds = 0
log_level = self.log.level
# lk: Disable up to INFO logging level for the segment below
# Rare: Disable up to INFO logging level for the segment below
self.log.setLevel(logging.ERROR)
self.log.info(f'= Progress: {perc:.02f}% ({processed_chunks}/{num_chunk_tasks}), '
f'Running for {rt_hours:02d}:{rt_minutes:02d}:{rt_seconds:02d}, '
@ -133,7 +141,7 @@ class DLManager(DLManagerReal):
f'/ {dl_unc_speed / 1024 / 1024:.02f} MiB/s (decompressed)')
self.log.info(f' + Disk\t- {w_speed / 1024 / 1024:.02f} MiB/s (write) / '
f'{r_speed / 1024 / 1024:.02f} MiB/s (read)')
# lk: Restore previous logging level
# Rare: Restore previous logging level
self.log.setLevel(log_level)
# send status update to back to instantiator (if queue exists)
@ -155,6 +163,29 @@ class DLManager(DLManagerReal):
except Exception as e:
self.log.warning(f'Failed to send status update to queue: {e!r}')
# Rare: queue of control signals
try:
signals: DLManagerSignals = self.signals_queue.get(timeout=0.5)
self.log.warning('Immediate stop requested!')
if signals.kill is True:
# lk: graceful but not what legendary does
self.running = False
# send conditions to unlock threads if they aren't already
for cond in self.conditions:
with cond:
cond.notify()
kill_request = True
break
# # lk: alternative way, but doesn't clean shm
# for i in range(self.max_workers):
# self.dl_worker_queue.put_nowait(TerminateWorkerTask())
#
# self.log.info('Waiting for installation to finish...')
# self.writer_queue.put_nowait(TerminateWorkerTask())
# raise KeyboardInterrupt
except queue.Empty:
pass
time.sleep(self.update_interval)
for i in range(self.max_workers):
@ -180,7 +211,7 @@ class DLManager(DLManagerReal):
self.log.warning(f'Thread did not terminate! {repr(t)}')
# clean up resume file
if self.resume_file:
if self.resume_file and not kill_request:
try:
os.remove(self.resume_file)
except OSError as e:
@ -194,4 +225,5 @@ class DLManager(DLManagerReal):
self.log.info('All done! Download manager quitting...')
# finally, exit the process.
exit(0)
# fmt: on
# fmt: on

3
rare/models/install.py
View file

@ -3,10 +3,11 @@ import platform as pf
from dataclasses import dataclass
from typing import List, Optional, Callable, Dict
from legendary.downloader.mp.manager import DLManager
from legendary.models.downloading import AnalysisResult, ConditionCheckResult
from legendary.models.game import Game, InstalledGame
from rare.lgndr.manager import DLManager
@dataclass
class InstallOptionsModel:

17
rare/utils/legendary_utils.py
View file

@ -5,9 +5,9 @@ from logging import getLogger
from PyQt5.QtCore import pyqtSignal, QObject, QRunnable, QStandardPaths
from legendary.core import LegendaryCore
from rare.lgndr.cli import LegendaryCLI
from rare.lgndr.api_monkeys import LgndrIndirectStatus
from rare.lgndr.api_arguments import LgndrVerifyGameArgs, LgndrUninstallGameArgs
from rare.lgndr.api_monkeys import LgndrIndirectStatus
from rare.lgndr.cli import LegendaryCLI
from rare.shared import LegendaryCoreSingleton, ArgumentsSingleton
from rare.utils import config_helper
@ -70,9 +70,7 @@ def update_manifest(app_name: str, core: LegendaryCore):
new_manifest = core.load_manifest(new_manifest_data)
logger.debug(f"Base urls: {base_urls}")
# save manifest with version name as well for testing/downgrading/etc.
core.lgd.save_manifest(
game.app_name, new_manifest_data, version=new_manifest.meta.build_version
)
core.lgd.save_manifest(game.app_name, new_manifest_data, version=new_manifest.meta.build_version)
class VerifyWorker(QRunnable):
@ -98,9 +96,9 @@ class VerifyWorker(QRunnable):
def run(self):
cli = LegendaryCLI(self.core)
status = LgndrIndirectStatus()
args = LgndrVerifyGameArgs(app_name=self.app_name,
indirect_status=status,
verify_stdout=self.status_callback)
args = LgndrVerifyGameArgs(
app_name=self.app_name, indirect_status=status, verify_stdout=self.status_callback
)
# lk: first pass, verify with the current manifest
repair_mode = False
@ -130,8 +128,7 @@ class VerifyWorker(QRunnable):
# lk: this could probably be cut down to what is relevant for this use-case and skip the `cli` call
igame = self.core.get_installed_game(self.app_name)
game = self.core.get_game(self.app_name, platform=igame.platform)
repair_file = os.path.join(self.core.lgd.get_tmp_path(), f'{self.app_name}.repair')
repair_file = os.path.join(self.core.lgd.get_tmp_path(), f"{self.app_name}.repair")
cli.clean_post_install(game=game, igame=igame, repair=True, repair_file=repair_file)
self.signals.result.emit(self.app_name, success, *result)