diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..cfe4bd12 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "legendary"] + path = legendary + url = https://github.com/dummerle/legendary + branch = rare diff --git a/custom_legendary/__init__.py b/custom_legendary/__init__.py deleted file mode 100644 index fc1bda59..00000000 --- a/custom_legendary/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Legendary!""" - -__version__ = '0.20.6' -__codename__ = 'Custom' diff --git a/custom_legendary/api/__init__.py b/custom_legendary/api/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/custom_legendary/api/egs.py b/custom_legendary/api/egs.py deleted file mode 100644 index 423b48a8..00000000 --- a/custom_legendary/api/egs.py +++ /dev/null @@ -1,167 +0,0 @@ -# !/usr/bin/env python -# coding: utf-8 - -import logging - -import requests -from requests.auth import HTTPBasicAuth - -from custom_legendary.models.exceptions import InvalidCredentialsError - - -class EPCAPI: - _user_agent = 'UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' - # required for the oauth request - _user_basic = '34a02cf8f4414e29b15921876da36f9a' - _pw_basic = 'daafbccc737745039dffe53d94fc76cf' - - _oauth_host = 'account-public-service-prod03.ol.epicgames.com' - _launcher_host = 'launcher-public-service-prod06.ol.epicgames.com' - _entitlements_host = 'entitlement-public-service-prod08.ol.epicgames.com' - _catalog_host = 'catalog-public-service-prod06.ol.epicgames.com' - _ecommerce_host = 'ecommerceintegration-public-service-ecomprod02.ol.epicgames.com' - _datastorage_host = 'datastorage-public-service-liveegs.live.use1a.on.epicgames.com' - _library_host = 'library-service.live.use1a.on.epicgames.com' - - def __init__(self, lc='en', cc='US'): - self.session = requests.session() - self.log = logging.getLogger('EPCAPI') - self.unauth_session = requests.session() - self.session.headers['User-Agent'] = self._user_agent - self.unauth_session.headers['User-Agent'] = self._user_agent - self._oauth_basic = HTTPBasicAuth(self._user_basic, self._pw_basic) - - self.access_token = None - self.user = None - - self.language_code = lc - self.country_code = cc - - def resume_session(self, session): - self.session.headers['Authorization'] = f'bearer {session["access_token"]}' - r = self.session.get(f'https://{self._oauth_host}/account/api/oauth/verify') - if r.status_code >= 500: - r.raise_for_status() - - j = r.json() - if 'errorMessage' in j: - self.log.warning(f'Login to EGS API failed with errorCode: {j["errorCode"]}') - raise InvalidCredentialsError(j['errorCode']) - - # update other data - session.update(j) - self.user = session - return self.user - - def start_session(self, refresh_token: str = None, exchange_token: str = None) -> dict: - if refresh_token: - params = dict(grant_type='refresh_token', - refresh_token=refresh_token, - token_type='eg1') - elif exchange_token: - params = dict(grant_type='exchange_code', - exchange_code=exchange_token, - token_type='eg1') - else: - raise ValueError('At least one token type must be specified!') - - r = self.session.post(f'https://{self._oauth_host}/account/api/oauth/token', - data=params, auth=self._oauth_basic) - # Only raise HTTP exceptions on server errors - if r.status_code >= 500: - r.raise_for_status() - - j = r.json() - if 'error' in j: - self.log.warning(f'Login to EGS API failed with errorCode: {j["errorCode"]}') - raise InvalidCredentialsError(j['errorCode']) - - self.user = j - self.session.headers['Authorization'] = f'bearer {self.user["access_token"]}' - return self.user - - def invalidate_session(self): # unused - r = self.session.delete(f'https://{self._oauth_host}/account/api/oauth/sessions/kill/{self.access_token}') - - def get_game_token(self): - r = self.session.get(f'https://{self._oauth_host}/account/api/oauth/exchange') - r.raise_for_status() - return r.json() - - def get_ownership_token(self, namespace, catalog_item_id): - user_id = self.user.get('account_id') - r = self.session.post(f'https://{self._ecommerce_host}/ecommerceintegration/api/public/' - f'platforms/EPIC/identities/{user_id}/ownershipToken', - data=dict(nsCatalogItemId=f'{namespace}:{catalog_item_id}')) - r.raise_for_status() - return r.content - - def get_game_assets(self, platform='Windows', label='Live'): - r = self.session.get(f'https://{self._launcher_host}/launcher/api/public/assets/{platform}', - params=dict(label=label)) - r.raise_for_status() - return r.json() - - def get_game_manifest(self, namespace, catalog_item_id, app_name, platform='Windows', label='Live'): - r = self.session.get(f'https://{self._launcher_host}/launcher/api/public/assets/v2/platform' - f'/{platform}/namespace/{namespace}/catalogItem/{catalog_item_id}/app' - f'/{app_name}/label/{label}') - r.raise_for_status() - return r.json() - - def get_user_entitlements(self): - user_id = self.user.get('account_id') - r = self.session.get(f'https://{self._entitlements_host}/entitlement/api/account/{user_id}/entitlements', - params=dict(start=0, count=5000)) - r.raise_for_status() - return r.json() - - def get_game_info(self, namespace, catalog_item_id): - r = self.session.get(f'https://{self._catalog_host}/catalog/api/shared/namespace/{namespace}/bulk/items', - params=dict(id=catalog_item_id, includeDLCDetails=True, includeMainGameDetails=True, - country=self.country_code, locale=self.language_code)) - r.raise_for_status() - return r.json().get(catalog_item_id, None) - - def get_library_items(self, include_metadata=True): - records = [] - r = self.session.get(f'https://{self._library_host}/library/api/public/items', - params=dict(includeMetadata=include_metadata)) - r.raise_for_status() - j = r.json() - records.extend(j['records']) - - # Fetch remaining library entries as long as there is a cursor - while cursor := j['responseMetadata'].get('nextCursor', None): - r = self.session.get(f'https://{self._library_host}/library/api/public/items', - params=dict(includeMetadata=include_metadata, cursor=cursor)) - r.raise_for_status() - j = r.json() - records.extend(j['records']) - - return records - - def get_user_cloud_saves(self, app_name='', manifests=False, filenames=None): - if app_name and manifests: - app_name += '/manifests/' - elif app_name: - app_name += '/' - - user_id = self.user.get('account_id') - - if filenames: - r = self.session.post(f'https://{self._datastorage_host}/api/v1/access/egstore/savesync/' - f'{user_id}/{app_name}', json=dict(files=filenames)) - else: - r = self.session.get(f'https://{self._datastorage_host}/api/v1/access/egstore/savesync/' - f'{user_id}/{app_name}') - r.raise_for_status() - return r.json() - - def create_game_cloud_saves(self, app_name, filenames): - return self.get_user_cloud_saves(app_name, filenames=filenames) - - def delete_game_cloud_save_file(self, path): - url = f'https://{self._datastorage_host}/api/v1/data/egstore/{path}' - r = self.session.delete(url) - r.raise_for_status() diff --git a/custom_legendary/cli.py b/custom_legendary/cli.py deleted file mode 100644 index be0adfc9..00000000 --- a/custom_legendary/cli.py +++ /dev/null @@ -1,1257 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import argparse -import csv -import datetime -import json -import logging -import os -import queue -import shlex -import subprocess -import time -import webbrowser -from distutils.util import strtobool -from getpass import getuser -from logging.handlers import QueueListener -from multiprocessing import freeze_support, Queue as MPQueue -from sys import exit, stdout - -from custom_legendary import __version__, __codename__ -from custom_legendary.core import LegendaryCore -from custom_legendary.models.exceptions import InvalidCredentialsError -from custom_legendary.models.game import SaveGameStatus -from custom_legendary.utils.cli import get_boolean_choice, sdl_prompt -from custom_legendary.utils.custom_parser import AliasedSubParsersAction - -# todo custom formatter for cli logger (clean info, highlighted error/warning) -logging.basicConfig( - format='[%(name)s] %(levelname)s: %(message)s', - level=logging.INFO -) -logger = logging.getLogger('cli') - - -class LegendaryCLI: - def __init__(self): - self.core = LegendaryCore() - self.logger = logging.getLogger('cli') - self.logging_queue = None - - def setup_threaded_logging(self): - self.logging_queue = MPQueue(-1) - shandler = logging.StreamHandler() - sformatter = logging.Formatter('[%(name)s] %(levelname)s: %(message)s') - shandler.setFormatter(sformatter) - ql = QueueListener(self.logging_queue, shandler) - ql.start() - return ql - - def auth(self, args): - if args.auth_delete: - self.core.lgd.invalidate_userdata() - return - - try: - logger.info('Testing existing login data if present...') - if self.core.login(): - logger.info('Stored credentials are still valid, if you wish to switch to a different ' - 'account, run "legendary auth --delete" and try again.') - return - except ValueError: - pass - except InvalidCredentialsError: - logger.error('Stored credentials were found but were no longer valid. Continuing with login...') - self.core.lgd.invalidate_userdata() - - if args.import_egs_auth: - # get appdata path on Linux - if not self.core.egl.appdata_path: - wine_pfx_users = None - lutris_wine_users = os.path.expanduser('~/Games/epic-games-store/drive_c/users') - if os.path.exists(lutris_wine_users): - logger.info(f'Found Lutris EGL WINE prefix at "{lutris_wine_users}"') - if args.yes or get_boolean_choice('Do you want to use the Lutris install?'): - wine_pfx_users = lutris_wine_users - - if not wine_pfx_users: - logger.info('Please enter the path to the Wine prefix that has EGL installed') - wine_pfx = input('Path [empty input to quit]: ').strip() - if not wine_pfx: - print('Empty input, quitting...') - exit(0) - if not os.path.exists(wine_pfx): - print('Path is invalid (does not exist)!') - exit(1) - wine_pfx_users = os.path.join(wine_pfx, 'drive_c/users') - - # todo instead of getuser() this should read from the user.reg in the WINE prefix - appdata_dir = os.path.join(wine_pfx_users, getuser(), - 'Local Settings/Application Data/EpicGamesLauncher', - 'Saved/Config/Windows') - if not os.path.exists(appdata_dir): - logger.error(f'Wine prefix does not have EGL appdata path at "{appdata_dir}"') - exit(0) - else: - logger.info(f'Using EGL appdata path at "{appdata_dir}"') - self.core.egl.appdata_path = appdata_dir - - logger.info('Importing login session from the Epic Launcher...') - try: - if self.core.auth_import(): - logger.info('Successfully imported login session from EGS!') - logger.info(f'Now logged in as user "{self.core.lgd.userdata["displayName"]}"') - return - else: - logger.warning('Login session from EGS seems to no longer be valid.') - exit(1) - except ValueError: - logger.error('No EGS login session found, please login manually.') - exit(1) - - exchange_token = '' - if not args.auth_code and not args.session_id: - # unfortunately the captcha stuff makes a complete CLI login flow kinda impossible right now... - print('Please login via the epic web login!') - webbrowser.open( - 'https://www.epicgames.com/id/login?redirectUrl=https%3A%2F%2Fwww.epicgames.com%2Fid%2Fapi%2Fredirect' - ) - print('If web page did not open automatically, please manually open the following URL: ' - 'https://www.epicgames.com/id/login?redirectUrl=https://www.epicgames.com/id/api/redirect') - sid = input('Please enter the "sid" value from the JSON response: ') - sid = sid.strip().strip('"') - exchange_token = self.core.auth_sid(sid) - elif args.session_id: - exchange_token = self.core.auth_sid(args.session_id) - elif args.auth_code: - exchange_token = args.auth_code - - if not exchange_token: - logger.fatal('No exchange token, cannot login.') - return - - if self.core.auth_code(exchange_token): - logger.info(f'Successfully logged in as "{self.core.lgd.userdata["displayName"]}"') - else: - logger.error('Login attempt failed, please see log for details.') - - def list_games(self, args): - logger.info('Logging in...') - if not self.core.login(): - logger.error('Login failed, cannot continue!') - exit(1) - logger.info('Getting game list... (this may take a while)') - games, dlc_list = self.core.get_game_and_dlc_list( - platform_override=args.platform_override, skip_ue=not args.include_ue - ) - # sort games and dlc by name - games = sorted(games, key=lambda x: x.app_title.lower()) - for citem_id in dlc_list.keys(): - dlc_list[citem_id] = sorted(dlc_list[citem_id], key=lambda d: d.app_title.lower()) - - if args.csv or args.tsv: - writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') - writer.writerow(['App name', 'App title', 'Version', 'Is DLC']) - for game in games: - writer.writerow((game.app_name, game.app_title, game.app_version, False)) - for dlc in dlc_list[game.asset_info.catalog_item_id]: - writer.writerow((dlc.app_name, dlc.app_title, dlc.app_version, True)) - return - - if args.json: - _out = [] - for game in games: - _j = vars(game) - _j['dlcs'] = [vars(dlc) for dlc in dlc_list[game.asset_info.catalog_item_id]] - _out.append(_j) - - print(json.dumps(_out, sort_keys=True, indent=2)) - return - - print('\nAvailable games:') - for game in games: - print(f' * {game.app_title} (App name: {game.app_name} | Version: {game.app_version})') - for dlc in dlc_list[game.asset_info.catalog_item_id]: - print(f' + {dlc.app_title} (App name: {dlc.app_name} | Version: {dlc.app_version})') - - print(f'\nTotal: {len(games)}') - - def list_installed(self, args): - if args.check_updates: - logger.info('Logging in to check for updates...') - if not self.core.login(): - logger.error('Login failed! Not checking for updates.') - else: - self.core.get_assets(True) - - games = sorted(self.core.get_installed_list(), - key=lambda x: x.title.lower()) - - versions = dict() - for game in games: - try: - versions[game.app_name] = self.core.get_asset(game.app_name).build_version - except ValueError: - logger.warning(f'Metadata for "{game.app_name}" is missing, the game may have been removed from ' - f'your account or not be in legendary\'s database yet, try rerunning the command ' - f'with "--check-updates".') - - if args.csv or args.tsv: - writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') - writer.writerow(['App name', 'App title', 'Installed version', 'Available version', - 'Update available', 'Install size', 'Install path']) - writer.writerows((game.app_name, game.title, game.version, versions[game.app_name], - versions[game.app_name] != game.version, game.install_size, game.install_path) - for game in games if game.app_name in versions) - return - - if args.json: - print(json.dumps([vars(g) for g in games], indent=2, sort_keys=True)) - return - - print('\nInstalled games:') - for game in games: - if game.install_size == 0: - logger.debug(f'Updating missing size for {game.app_name}') - m = self.core.load_manifest(self.core.get_installed_manifest(game.app_name)[0]) - game.install_size = sum(fm.file_size for fm in m.file_manifest_list.elements) - self.core.install_game(game) - - print(f' * {game.title} (App name: {game.app_name} | Version: {game.version} | ' - f'{game.install_size / (1024 * 1024 * 1024):.02f} GiB)') - if args.include_dir: - print(f' + Location: {game.install_path}') - if not os.path.exists(game.install_path): - print(f' ! Game does no longer appear to be installed (directory "{game.install_path}" missing)!') - elif game.app_name in versions and versions[game.app_name] != game.version: - print(f' -> Update available! Installed: {game.version}, Latest: {versions[game.app_name]}') - - print(f'\nTotal: {len(games)}') - - def list_files(self, args): - if args.platform_override: - args.force_download = True - - if not args.override_manifest and not args.app_name: - print('You must provide either a manifest url/path or app name!') - return - - # check if we even need to log in - if args.override_manifest: - logger.info(f'Loading manifest from "{args.override_manifest}"') - manifest_data, _ = self.core.get_uri_manifest(args.override_manifest) - elif self.core.is_installed(args.app_name) and not args.force_download: - logger.info(f'Loading installed manifest for "{args.app_name}"') - manifest_data, _ = self.core.get_installed_manifest(args.app_name) - else: - logger.info(f'Logging in and downloading manifest for {args.app_name}') - if not self.core.login(): - logger.error('Login failed! Cannot continue with download process.') - exit(1) - game = self.core.get_game(args.app_name, update_meta=True) - if not game: - logger.fatal(f'Could not fetch metadata for "{args.app_name}" (check spelling/account ownership)') - exit(1) - manifest_data, _ = self.core.get_cdn_manifest(game, platform_override=args.platform_override) - - manifest = self.core.load_manifest(manifest_data) - files = sorted(manifest.file_manifest_list.elements, - key=lambda a: a.filename.lower()) - - if args.install_tag: - files = [fm for fm in files if args.install_tag in fm.install_tags] - - if args.hashlist: - for fm in files: - print(f'{fm.hash.hex()} *{fm.filename}') - elif args.csv or args.tsv: - writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') - writer.writerow(['path', 'hash', 'size', 'install_tags']) - writer.writerows((fm.filename, fm.hash.hex(), fm.file_size, '|'.join(fm.install_tags)) for fm in files) - elif args.json: - _files = [] - for fm in files: - _files.append(dict( - filename=fm.filename, - sha_hash=fm.hash.hex(), - install_tags=fm.install_tags, - file_size=fm.file_size, - flags=fm.flags, - )) - print(json.dumps(_files, sort_keys=True, indent=2)) - else: - install_tags = set() - for fm in files: - print(fm.filename) - for t in fm.install_tags: - install_tags.add(t) - if install_tags: - # use the log output so this isn't included when piping file list into file - logger.info(f'Install tags: {", ".join(sorted(install_tags))}') - - def list_saves(self, args): - if not self.core.login(): - logger.error('Login failed! Cannot continue with download process.') - exit(1) - # update game metadata - logger.debug('Refreshing games list...') - _ = self.core.get_game_and_dlc_list(update_assets=True) - # then get the saves - logger.info('Getting list of saves...') - saves = self.core.get_save_games(args.app_name) - last_app = '' - print('Save games:') - for save in sorted(saves, key=lambda a: a.app_name + a.manifest_name): - if save.app_name != last_app: - game_title = self.core.get_game(save.app_name).app_title - last_app = save.app_name - print(f'- {game_title} ("{save.app_name}")') - print(' +', save.manifest_name) - - def download_saves(self, args): - if not self.core.login(): - logger.error('Login failed! Cannot continue with download process.') - exit(1) - logger.info(f'Downloading saves to "{self.core.get_default_install_dir()}"') - self.core.download_saves(args.app_name) - - def sync_saves(self, args): - if not self.core.login(): - logger.error('Login failed! Cannot continue with download process.') - exit(1) - - igames = self.core.get_installed_list() - if args.app_name: - igame = self.core.get_installed_game(args.app_name) - if not igame: - logger.fatal(f'Game not installed: {args.app_name}') - exit(1) - igames = [igame] - - # check available saves - saves = self.core.get_save_games() - latest_save = dict() - - for save in sorted(saves, key=lambda a: a.datetime): - latest_save[save.app_name] = save - - logger.info(f'Got {len(latest_save)} remote save game(s)') - - # evaluate current save state for each game. - for igame in igames: - game = self.core.get_game(igame.app_name) - if not game or not game.supports_cloud_saves: - if igame.app_name in latest_save: - # this should never happen unless cloud save support was removed from a game - logger.warning(f'{igame.app_name} has remote save(s) but does not support cloud saves?!') - continue - - logger.info(f'Checking "{igame.title}" ({igame.app_name})') - # override save path only if app name is specified - if args.app_name and args.save_path: - logger.info(f'Overriding save path with "{args.save_path}"...') - igame.save_path = args.save_path - self.core.lgd.set_installed_game(igame.app_name, igame) - - # if there is no saved save path, try to get one - if not igame.save_path: - save_path = self.core.get_save_path(igame.app_name) - - # ask user if path is correct if computing for the first time - logger.info(f'Computed save path: "{save_path}"') - - if '%' in save_path or '{' in save_path: - logger.warning('Path contains unprocessed variables, please enter the correct path manually.') - yn = False - else: - yn = get_boolean_choice('Is this correct?') - - if not yn: - save_path = input('Please enter the correct path (leave empty to skip): ') - if not save_path: - logger.info('Empty input, skipping...') - continue - - if not os.path.exists(save_path): - os.makedirs(save_path) - igame.save_path = save_path - self.core.lgd.set_installed_game(igame.app_name, igame) - - res, (dt_l, dt_r) = self.core.check_savegame_state(igame.save_path, latest_save.get(igame.app_name)) - - if res == SaveGameStatus.NO_SAVE: - logger.info('No cloud or local savegame found.') - continue - - if res == SaveGameStatus.SAME_AGE and not (args.force_upload or args.force_download): - logger.info(f'Save game for "{igame.title}" is up to date, skipping...') - continue - - if (res == SaveGameStatus.REMOTE_NEWER and not args.force_upload) or args.force_download: - if res == SaveGameStatus.REMOTE_NEWER: # only print this info if not forced - logger.info(f'Cloud save for "{igame.title}" is newer:') - logger.info(f'- Cloud save date: {dt_r.strftime("%Y-%m-%d %H:%M:%S")}') - if dt_l: - logger.info(f'- Local save date: {dt_l.strftime("%Y-%m-%d %H:%M:%S")}') - else: - logger.info('- Local save date: N/A') - - if args.upload_only: - logger.info('Save game downloading is disabled, skipping...') - continue - - if not args.yes and not args.force_download: - if not get_boolean_choice(f'Download cloud save?'): - logger.info('Not downloading...') - continue - - logger.info('Downloading remote savegame...') - self.core.download_saves(igame.app_name, save_dir=igame.save_path, clean_dir=True, - manifest_name=latest_save[igame.app_name].manifest_name) - elif res == SaveGameStatus.LOCAL_NEWER or args.force_upload: - if res == SaveGameStatus.LOCAL_NEWER: - logger.info(f'Local save for "{igame.title}" is newer') - if dt_r: - logger.info(f'- Cloud save date: {dt_r.strftime("%Y-%m-%d %H:%M:%S")}') - else: - logger.info('- Cloud save date: N/A') - logger.info(f'- Local save date: {dt_l.strftime("%Y-%m-%d %H:%M:%S")}') - - if args.download_only: - logger.info('Save game uploading is disabled, skipping...') - continue - - if not args.yes and not args.force_upload: - if not get_boolean_choice(f'Upload local save?'): - logger.info('Not uploading...') - continue - logger.info('Uploading local savegame...') - self.core.upload_save(igame.app_name, igame.save_path, dt_l, args.disable_filters) - - def launch_game(self, args, extra): - app_name = args.app_name - igame = self.core.get_installed_game(app_name) - if not igame: - logger.error(f'Game {app_name} is not currently installed!') - exit(1) - - if igame.is_dlc: - logger.error(f'{app_name} is DLC; please launch the base game instead!') - exit(1) - - if not os.path.exists(igame.install_path): - logger.fatal(f'Install directory "{igame.install_path}" appears to be deleted, cannot launch {app_name}!') - exit(1) - - if args.reset_defaults: - logger.info(f'Removing configuration section for "{app_name}"...') - self.core.lgd.config.remove_section(app_name) - return - - # override with config value - args.offline = self.core.is_offline_game(app_name) or args.offline - if not args.offline: - logger.info('Logging in...') - if not self.core.login(): - logger.error('Login failed, cannot continue!') - exit(1) - - if not args.skip_version_check and not self.core.is_noupdate_game(app_name): - logger.info('Checking for updates...') - try: - latest = self.core.get_asset(app_name, update=True) - except ValueError: - logger.fatal(f'Metadata for "{app_name}" does not exist, cannot launch!') - exit(1) - - if latest.build_version != igame.version: - logger.error('Game is out of date, please update or launch with update check skipping!') - exit(1) - - params, cwd, env = self.core.get_launch_parameters(app_name=app_name, offline=args.offline, - extra_args=extra, user=args.user_name_override, - wine_bin=args.wine_bin, wine_pfx=args.wine_pfx, - language=args.language, wrapper=args.wrapper, - disable_wine=args.no_wine, - executable_override=args.executable_override) - - if args.set_defaults: - self.core.lgd.config[app_name] = dict() - # we have to do this if-cacophony here because an empty value is still - # valid and could cause issues when relying on config.get()'s fallback - if args.offline: - self.core.lgd.config[app_name]['offline'] = 'true' - if args.skip_version_check: - self.core.lgd.config[app_name]['skip_update_check'] = 'true' - if extra: - self.core.lgd.config[app_name]['start_params'] = shlex.join(extra) - if args.wine_bin: - self.core.lgd.config[app_name]['wine_executable'] = args.wine_bin - if args.wine_pfx: - self.core.lgd.config[app_name]['wine_prefix'] = args.wine_pfx - if args.no_wine: - self.core.lgd.config[app_name]['no_wine'] = 'true' - if args.language: - self.core.lgd.config[app_name]['language'] = args.language - if args.wrapper: - self.core.lgd.config[app_name]['wrapper'] = args.wrapper - - if args.dry_run: - logger.info(f'Not Launching {app_name} (dry run)') - logger.info(f'Launch parameters: {shlex.join(params)}') - logger.info(f'Working directory: {cwd}') - if env: - logger.info(f'Environment overrides: {env}') - else: - logger.info(f'Launching {app_name}...') - logger.debug(f'Launch parameters: {shlex.join(params)}') - logger.debug(f'Working directory: {cwd}') - if env: - logger.debug(f'Environment overrides: {env}') - subprocess.Popen(params, cwd=cwd, env=env) - - def install_game(self, args): - if args.subparser_name == 'download': - logger.info('Setting --no-install flag since "download" command was used') - args.no_install = True - elif args.subparser_name == 'repair': - args.repair_mode = True - - if args.update_only: - if not self.core.is_installed(args.app_name): - logger.error(f'Update requested for "{args.app_name}", but app not installed!') - exit(1) - - status_queue = MPQueue() - logger.info('Preparing download...') - try: - dlm, analysis, game, igame, repair, repair_file, res = self.core.prepare_download( - app_name=args.app_name, - base_path=args.base_path, - force=args.force, - no_install=args.no_install, - status_q=status_queue, - max_shm=args.shared_memory, - max_workers=args.max_workers, - game_folder=args.game_folder, - disable_patching=args.disable_patching, - override_manifest=args.override_manifest, - override_old_manifest=args.override_old_manifest, - override_base_url=args.override_base_url, - platform_override=args.platform_override, - file_prefix_filter=args.file_prefix, - file_exclude_filter=args.file_exclude_prefix, - file_install_tag=args.install_tag, - dl_optimizations=args.order_opt, - dl_timeout=args.dl_timeout, - repair=args.repair_mode, - repair_use_latest=args.repair_and_update, - ignore_space_req=args.ignore_space, - disable_delta=args.disable_delta, - override_delta_manifest=args.override_delta_manifest, - reset_sdl=args.reset_sdl, - sdl_prompt=sdl_prompt) - except Exception as e: - logger.fatal(e) - exit(1) - - if res.failures: - for i in res.failures: - logger.fatal(i) - exit(1) - - if res.warnings: - for warn in res.warnings: - logger.warning(warn) - - logger.info(f'Install size: {analysis.install_size / 1024 / 1024:.02f} MiB') - compression = (1 - (analysis.dl_size / analysis.uncompressed_dl_size)) * 100 - logger.info(f'Download size: {analysis.dl_size / 1024 / 1024:.02f} MiB ' - f'(Compression savings: {compression:.01f}%)') - logger.info(f'Reusable size: {analysis.reuse_size / 1024 / 1024:.02f} MiB (chunks) / ' - f'{analysis.unchanged / 1024 / 1024:.02f} MiB (unchanged / skipped)') - - if not args.yes: - if not get_boolean_choice(f'Do you wish to install "{igame.title}"?'): - print('Aborting...') - exit(0) - - logger.info('Downloads are resumable, you can interrupt the download with ' - 'CTRL-C and resume it using the same command later on.') - - start_t = time.time() - - try: - # set up logging stuff (should be moved somewhere else later) - dlm.logging_queue = self.logging_queue - dlm.proc_debug = args.dlm_debug - - dlm.start() - time.sleep(1) - while dlm.is_alive(): - try: - status = status_queue.get(timeout=0.1) - logger.info( - f'= Progress: {status.progress:.02f}% ({status.processed_chunks}/{status.chunk_tasks}), ' - f'Running for {str(datetime.timedelta(seconds=status.runtime))}, ' - f'ETA: {str(datetime.timedelta(seconds=status.estimated_time_left))}') - logger.info(f' - Downloaded: {status.total_downloaded / 1024 / 1024:.02f} MiB, ' - f'Written: {status.total_written / 1024 / 1024:.02f} MiB') - logger.info(f' - Cache usage: {status.cache_usage} MiB, active tasks: {status.active_tasks}') - logger.info(f' + Download\t- {status.download_speed / 1024 / 1024:.02f} MiB/s (raw) ' - f'/ {status.download_decompressed_speed / 1024 / 1024:.02f} MiB/s (decompressed)') - logger.info(f' + Disk\t- {status.write_speed / 1024 / 1024:.02f} MiB/s (write) / ' - f'{status.read_speed / 1024 / 1024:.02f} MiB/s (read)') - except queue.Empty: - pass - dlm.join() - except Exception as e: - end_t = time.time() - logger.info(f'Installation failed after {end_t - start_t:.02f} seconds.') - logger.warning(f'The following exception occurred while waiting for the downloader to finish: {e!r}. ' - f'Try restarting the process, the resume file will be used to start where it failed. ' - f'If it continues to fail please open an issue on GitHub.') - else: - end_t = time.time() - if not args.no_install: - # Allow setting savegame directory at install time so sync-saves will work immediately - if game.supports_cloud_saves and args.save_path: - igame.save_path = args.save_path - - postinstall = self.core.install_game(igame) - if postinstall: - self._handle_postinstall(postinstall, igame, yes=args.yes) - - dlcs = self.core.get_dlc_for_game(game.app_name) - if dlcs: - print('The following DLCs are available for this game:') - for dlc in dlcs: - print(f' - {dlc.app_title} (App name: {dlc.app_name}, version: {dlc.app_version})') - print('Manually installing DLCs works the same; just use the DLC app name instead.') - - install_dlcs = True - if not args.yes: - if not get_boolean_choice(f'Do you wish to automatically install DLCs?'): - install_dlcs = False - - if install_dlcs: - _yes, _app_name = args.yes, args.app_name - args.yes = True - for dlc in dlcs: - args.app_name = dlc.app_name - self.install_game(args) - args.yes, args.app_name = _yes, _app_name - - if game.supports_cloud_saves and not game.is_dlc: - # todo option to automatically download saves after the installation - # args does not have the required attributes for sync_saves in here, - # not sure how to solve that elegantly. - logger.info('This game supports cloud saves, syncing is handled by the "sync-saves" command.') - logger.info(f'To download saves for this game run "legendary sync-saves {args.app_name}"') - - self.core.clean_post_install(game=game, igame=igame, repair=repair, repair_file=repair_file) - - logger.info(f'Finished installation process in {end_t - start_t:.02f} seconds.') - - def _handle_postinstall(self, postinstall, igame, yes=False): - print('This game lists the following prequisites to be installed:') - print(f'- {postinstall["name"]}: {" ".join((postinstall["path"], postinstall["args"]))}') - if os.name == 'nt': - if yes: - c = 'n' # we don't want to launch anything, just silent install. - else: - choice = input('Do you wish to install the prerequisites? ([y]es, [n]o, [i]gnore): ') - c = choice.lower()[0] - - if c == 'i': # just set it to installed - print('Marking prerequisites as installed...') - self.core.prereq_installed(igame.app_name) - elif c == 'y': # set to installed and launch installation - print('Launching prerequisite executable..') - self.core.prereq_installed(igame.app_name) - req_path, req_exec = os.path.split(postinstall['path']) - work_dir = os.path.join(igame.install_path, req_path) - fullpath = os.path.join(work_dir, req_exec) - subprocess.Popen([fullpath, postinstall['args']], cwd=work_dir) - else: - logger.info('Automatic installation not available on Linux.') - - def uninstall_game(self, args): - igame = self.core.get_installed_game(args.app_name) - if not igame: - logger.error(f'Game {args.app_name} not installed, cannot uninstall!') - exit(0) - if igame.is_dlc: - logger.error('Uninstalling DLC is not supported.') - exit(1) - - if not args.yes: - if not get_boolean_choice(f'Do you wish to uninstall "{igame.title}"?', default=False): - print('Aborting...') - exit(0) - - try: - # Remove DLC first so directory is empty when game uninstall runs - dlcs = self.core.get_dlc_for_game(igame.app_name) - for dlc in dlcs: - if (idlc := self.core.get_installed_game(dlc.app_name)) is not None: - logger.info(f'Uninstalling DLC "{dlc.app_name}"...') - self.core.uninstall_game(idlc, delete_files=not args.keep_files) - - logger.info(f'Removing "{igame.title}" from "{igame.install_path}"...') - self.core.uninstall_game(igame, delete_files=not args.keep_files, delete_root_directory=True) - logger.info('Game has been uninstalled.') - except Exception as e: - logger.warning(f'Removing game failed: {e!r}, please remove {igame.install_path} manually.') - - def output_progress(self, num, total): - stdout.write(f'Verification progress: {num}/{total} ({num * 100 / total:.01f}%)\t\r') - stdout.flush() - - def verify_game(self, args, print_command=True): - try: - self.core.verify_game(app_name=args.app_name, callback=self.output_progress) - except Exception as e: - logger.error(e) - if print_command: - logger.info(f'Run "legendary repair {args.app_name}" to repair your game installation.') - - def import_game(self, args): - # make sure path is absolute - args.app_path = os.path.abspath(args.app_path) - - if not os.path.exists(args.app_path): - logger.error(f'Specified path "{args.app_path}" does not exist!') - exit(1) - - if self.core.is_installed(args.app_name): - logger.error('Game is already installed!') - exit(0) - - if not self.core.login(): - logger.error('Log in failed!') - exit(1) - - # do some basic checks - game = self.core.get_game(args.app_name, update_meta=True) - if not game: - logger.fatal(f'Did not find game "{args.app_name}" on account.') - exit(1) - - # get everything needed for import from core, then run additional checks. - manifest, igame = self.core.import_game(game, args.app_path) - exe_path = os.path.join(args.app_path, manifest.meta.launch_exe.lstrip('/')) - # check if most files at least exist or if user might have specified the wrong directory - total = len(manifest.file_manifest_list.elements) - found = sum(os.path.exists(os.path.join(args.app_path, f.filename)) - for f in manifest.file_manifest_list.elements) - ratio = found / total - - if not os.path.exists(exe_path and not args.disable_check): - logger.error(f'Game executable could not be found at "{exe_path}", ' - f'please verify that the specified path is correct.') - exit(1) - - if ratio < 0.95: - logger.warning('Some files are missing from the game installation, install may not ' - 'match latest Epic Games Store version or might be corrupted.') - else: - logger.info('Game install appears to be complete.') - - self.core.install_game(igame) - if igame.needs_verification: - logger.info(f'NOTE: The game installation will have to be verified before it can be updated ' - f'with legendary. Run "legendary repair {args.app_name}" to do so.') - else: - logger.info(f'Installation had Epic Games Launcher metadata for version "{igame.version}", ' - f'verification will not be required.') - logger.info('Game has been imported.') - - def egs_sync(self, args): - if args.unlink: - logger.info('Unlinking and resetting EGS and LGD sync...') - self.core.lgd.config.remove_option('Legendary', 'egl_programdata') - self.core.lgd.config.remove_option('Legendary', 'egl_sync') - # remove EGL GUIDs from all games, DO NOT remove .egstore folders because that would fuck things up. - for igame in self.core.get_installed_list(): - igame.egl_guid = '' - self.core.install_game(igame) - # todo track which games were imported, remove those from LGD and exported ones from EGL - logger.info('NOTE: Games have not been removed from the Epic Games Launcher or Legendary.') - logger.info('Games will not be removed from EGL or Legendary if it was removed from the other launcher.') - return - elif args.disable_sync: - logger.info('Disabling EGS/LGD sync...') - self.core.lgd.config.remove_option('Legendary', 'egl_sync') - return - - if not self.core.lgd.assets: - logger.error('Legendary is missing game metadata, please login (if not already) and use the ' - '"status" command to fetch necessary information to set-up syncing.') - return - - if not self.core.egl.programdata_path: - if not args.egl_manifest_path and not args.egl_wine_prefix: - # search default Lutris install path - lutris_data_path = os.path.expanduser('~/Games/epic-games-store/drive_c/ProgramData' - '/Epic/EpicGamesLauncher/Data') - egl_path = None - if os.path.exists(lutris_data_path): - logger.info(f'Found Lutris EGL install at "{lutris_data_path}"') - - if args.yes or get_boolean_choice('Do you want to use the Lutris install?'): - egl_path = os.path.join(lutris_data_path, 'Manifests') - if not os.path.exists(egl_path): - print('EGL Data path exists but Manifests directory is missing, creating...') - os.makedirs(egl_path) - - if not egl_path: - print('EGL path not found, please manually provide the path to the WINE prefix it is installed in') - egl_path = input('Path [empty input to quit]: ').strip() - if not egl_path: - print('Empty input, quitting...') - exit(0) - if not os.path.exists(egl_path): - print('Path is invalid (does not exist)!') - exit(1) - egl_data_path = os.path.join(egl_path, 'drive_c/ProgramData/Epic/EpicGamesLauncher/Data') - egl_path = os.path.join(egl_data_path, 'Manifests') - if not os.path.exists(egl_path): - if not os.path.exists(egl_data_path): - print('Invalid path (wrong directory, WINE prefix, or EGL not installed/launched)') - exit(1) - print('EGL Data path exists but Manifests directory is missing, creating...') - os.makedirs(egl_path) - - if not os.listdir(egl_path): - logger.warning('Folder is empty, this may be fine if nothing has been installed yet.') - self.core.egl.programdata_path = egl_path - self.core.lgd.config.set('Legendary', 'egl_programdata', egl_path) - elif args.egl_wine_prefix: - egl_data_path = os.path.join(args.egl_wine_prefix, - 'drive_c/ProgramData/Epic/EpicGamesLauncher/Data') - egl_path = os.path.join(egl_data_path, 'Manifests') - if not os.path.exists(egl_path): - if not os.path.exists(egl_data_path): - print('Invalid path (wrong directory, WINE prefix, or EGL not installed/launched)') - exit(1) - print('EGL Data path exists but Manifests directory is missing, creating...') - os.makedirs(egl_path) - - if not os.listdir(egl_path): - logger.warning('Folder is empty, this may be fine if nothing has been installed yet.') - self.core.egl.programdata_path = egl_path - self.core.lgd.config.set('Legendary', 'egl_programdata', egl_path) - else: - if not os.path.exists(args.egl_manifest_path): - logger.fatal('Path specified via --egl-manifest-path does not exist') - exit(1) - self.core.egl.programdata_path = args.egl_manifest_path - self.core.lgd.config.set('Legendary', 'egl_programdata', args.egl_manifest_path) - - logger.debug(f'Using EGL ProgramData path "{self.core.egl.programdata_path}"...') - logger.info('Reading EGL game manifests...') - - if not args.export_only: - print('\nChecking for importable games...') - importable = self.core.egl_get_importable() - if importable: - print('The following games are importable (EGL -> Legendary):') - for egl_game in importable: - print(' *', egl_game.app_name, '-', egl_game.display_name) - - print('\nNote: Only games that are also in Legendary\'s database are listed, ' - 'if anything is missing run "list-games" first to update it.') - - if args.yes or get_boolean_choice('Do you want to import the games from EGL?'): - for egl_game in importable: - logger.info(f'Importing "{egl_game.display_name}"...') - self.core.egl_import(egl_game.app_name) - else: - print('Nothing to import.') - - if not args.import_only: - print('\nChecking for exportable games...') - exportable = self.core.egl_get_exportable() - if exportable: - print('The following games are exportable (Legendary -> EGL)') - for lgd_game in exportable: - print(' *', lgd_game.app_name, '-', lgd_game.title) - - if args.yes or get_boolean_choice('Do you want to export the games to EGL?'): - for lgd_game in exportable: - logger.info(f'Exporting "{lgd_game.title}"...') - self.core.egl_export(lgd_game.app_name) - else: - print('Nothing to export.') - - print('\nChecking automatic sync...') - if not self.core.egl_sync_enabled and not args.one_shot: - if not args.enable_sync: - args.enable_sync = args.yes or get_boolean_choice('Enable automatic synchronization?') - if not args.enable_sync: # if user chooses no, still run the sync once - self.core.egl_sync() - self.core.lgd.config.set('Legendary', 'egl_sync', str(args.enable_sync)) - else: - self.core.egl_sync() - - def status(self, args): - if not args.offline: - try: - if not self.core.login(): - logger.error('Log in failed!') - exit(1) - except ValueError: - pass - - if not self.core.lgd.userdata: - user_name = '' - args.offline = True - else: - user_name = self.core.lgd.userdata['displayName'] - - games_available = len(self.core.get_game_list(update_assets=not args.offline)) - games_installed = len(self.core.get_installed_list()) - if args.json: - print(json.dumps(dict( - account=user_name, - games_available=games_available, - games_installed=games_installed, - egl_sync_enabled=self.core.egl_sync_enabled, - config_directory=self.core.lgd.path - ), indent=2, sort_keys=True)) - return - - print(f'Epic account: {user_name}') - print(f'Games available: {games_available}') - print(f'Games installed: {games_installed}') - print(f'EGL Sync enabled: {self.core.egl_sync_enabled}') - print(f'Config directory: {self.core.lgd.path}') - - def cleanup(self, args): - before = self.core.lgd.get_dir_size() - # delete metadata - logger.debug('Removing app metadata...') - app_names = set(g.app_name for g in self.core.get_assets(update_assets=False)) - self.core.lgd.clean_metadata(app_names) - - if not args.keep_manifests: - logger.debug('Removing manifests...') - installed = [(ig.app_name, ig.version) for ig in self.core.get_installed_list()] - installed.extend((ig.app_name, ig.version) for ig in self.core.get_installed_dlc_list()) - self.core.lgd.clean_manifests(installed) - - logger.debug('Removing tmp data') - self.core.lgd.clean_tmp_data() - - after = self.core.lgd.get_dir_size() - logger.info(f'Cleanup complete! Removed {(before - after) / 1024 / 1024:.02f} MiB.') - - -def main(): - parser = argparse.ArgumentParser(description=f'Legendary v{__version__} - "{__codename__}"') - parser.register('action', 'parsers', AliasedSubParsersAction) - - # general arguments - parser.add_argument('-v', '--debug', dest='debug', action='store_true', help='Set loglevel to debug') - parser.add_argument('-y', '--yes', dest='yes', action='store_true', help='Default to yes for all prompts') - parser.add_argument('-V', '--version', dest='version', action='store_true', help='Print version and exit') - - # all the commands - subparsers = parser.add_subparsers(title='Commands', dest='subparser_name') - auth_parser = subparsers.add_parser('auth', help='Authenticate with EPIC') - install_parser = subparsers.add_parser('install', help='Download a game', - aliases=('download', 'update', 'repair'), - usage='%(prog)s [options]', - description='Aliases: download, update') - uninstall_parser = subparsers.add_parser('uninstall', help='Uninstall (delete) a game') - launch_parser = subparsers.add_parser('launch', help='Launch a game', usage='%(prog)s [options]', - description='Note: additional arguments are passed to the game') - list_parser = subparsers.add_parser('list-games', help='List available (installable) games') - list_installed_parser = subparsers.add_parser('list-installed', help='List installed games') - list_files_parser = subparsers.add_parser('list-files', help='List files in manifest') - list_saves_parser = subparsers.add_parser('list-saves', help='List available cloud saves') - download_saves_parser = subparsers.add_parser('download-saves', help='Download all cloud saves') - sync_saves_parser = subparsers.add_parser('sync-saves', help='Sync cloud saves') - verify_parser = subparsers.add_parser('verify-game', help='Verify a game\'s local files') - import_parser = subparsers.add_parser('import-game', help='Import an already installed game') - egl_sync_parser = subparsers.add_parser('egl-sync', help='Setup or run Epic Games Launcher sync') - status_parser = subparsers.add_parser('status', help='Show legendary status information') - clean_parser = subparsers.add_parser('cleanup', help='Remove old temporary, metadata, and manifest files') - - install_parser.add_argument('app_name', help='Name of the app', metavar='') - uninstall_parser.add_argument('app_name', help='Name of the app', metavar='') - launch_parser.add_argument('app_name', help='Name of the app', metavar='') - list_files_parser.add_argument('app_name', nargs='?', metavar='', - help='Name of the app (optional)') - list_saves_parser.add_argument('app_name', nargs='?', metavar='', default='', - help='Name of the app (optional)') - download_saves_parser.add_argument('app_name', nargs='?', metavar='', default='', - help='Name of the app (optional)') - sync_saves_parser.add_argument('app_name', nargs='?', metavar='', default='', - help='Name of the app (optional)') - verify_parser.add_argument('app_name', help='Name of the app', metavar='') - import_parser.add_argument('app_name', help='Name of the app', metavar='') - import_parser.add_argument('app_path', help='Path where the game is installed', - metavar='') - - auth_parser.add_argument('--import', dest='import_egs_auth', action='store_true', - help='Import Epic Games Launcher authentication data (logs out of EGL)') - auth_parser.add_argument('--code', dest='auth_code', action='store', metavar='', - help='Use specified exchange code instead of interactive authentication') - auth_parser.add_argument('--sid', dest='session_id', action='store', metavar='', - help='Use specified session id instead of interactive authentication') - auth_parser.add_argument('--delete', dest='auth_delete', action='store_true', - help='Remove existing authentication (log out)') - - install_parser.add_argument('--base-path', dest='base_path', action='store', metavar='', - help='Path for game installations (defaults to ~/legendary)') - install_parser.add_argument('--game-folder', dest='game_folder', action='store', metavar='', - help='Folder for game installation (defaults to folder specified in metadata)') - install_parser.add_argument('--max-shared-memory', dest='shared_memory', action='store', metavar='', - type=int, help='Maximum amount of shared memory to use (in MiB), default: 1 GiB') - install_parser.add_argument('--max-workers', dest='max_workers', action='store', metavar='', - type=int, help='Maximum amount of download workers, default: min(2 * CPUs, 16)') - install_parser.add_argument('--manifest', dest='override_manifest', action='store', metavar='', - help='Manifest URL or path to use instead of the CDN one (e.g. for downgrading)') - install_parser.add_argument('--old-manifest', dest='override_old_manifest', action='store', metavar='', - help='Manifest URL or path to use as the old one (e.g. for testing patching)') - install_parser.add_argument('--delta-manifest', dest='override_delta_manifest', action='store', metavar='', - help='Manifest URL or path to use as the delta one (e.g. for testing)') - install_parser.add_argument('--base-url', dest='override_base_url', action='store', metavar='', - help='Base URL to download from (e.g. to test or switch to a different CDNs)') - install_parser.add_argument('--force', dest='force', action='store_true', - help='Download all files / ignore existing (overwrite)') - install_parser.add_argument('--disable-patching', dest='disable_patching', action='store_true', - help='Do not attempt to patch existing installation (download entire changed files)') - install_parser.add_argument('--download-only', '--no-install', dest='no_install', action='store_true', - help='Do not install app and do not run prerequisite installers after download') - install_parser.add_argument('--update-only', dest='update_only', action='store_true', - help='Only update, do not do anything if specified app is not installed') - install_parser.add_argument('--dlm-debug', dest='dlm_debug', action='store_true', - help='Set download manager and worker processes\' loglevel to debug') - install_parser.add_argument('--platform', dest='platform_override', action='store', metavar='', - type=str, help='Platform override for download (also sets --no-install)') - install_parser.add_argument('--prefix', dest='file_prefix', action='append', metavar='', - help='Only fetch files whose path starts with (case insensitive)') - install_parser.add_argument('--exclude', dest='file_exclude_prefix', action='append', metavar='', - type=str, help='Exclude files starting with (case insensitive)') - install_parser.add_argument('--install-tag', dest='install_tag', action='append', metavar='', - type=str, help='Only download files with the specified install tag') - install_parser.add_argument('--enable-reordering', dest='order_opt', action='store_true', - help='Enable reordering optimization to reduce RAM requirements ' - 'during download (may have adverse results for some titles)') - install_parser.add_argument('--dl-timeout', dest='dl_timeout', action='store', metavar='', type=int, - help='Connection timeout for downloader (default: 10 seconds)') - install_parser.add_argument('--save-path', dest='save_path', action='store', metavar='', - help='Set save game path to be used for sync-saves') - install_parser.add_argument('--repair', dest='repair_mode', action='store_true', - help='Repair installed game by checking and redownloading corrupted/missing files') - install_parser.add_argument('--repair-and-update', dest='repair_and_update', action='store_true', - help='Update game to the latest version when repairing') - install_parser.add_argument('--ignore-free-space', dest='ignore_space', action='store_true', - help='Do not abort if not enough free space is available') - install_parser.add_argument('--disable-delta-manifests', dest='disable_delta', action='store_true', - help='Do not use delta manifests when updating (may increase download size)') - install_parser.add_argument('--reset-sdl', dest='reset_sdl', action='store_true', - help='Reset selective downloading choices (requires repair to download new components)') - - uninstall_parser.add_argument('--keep-files', dest='keep_files', action='store_true', - help='Keep files but remove game from Legendary database') - - launch_parser.add_argument('--offline', dest='offline', action='store_true', - default=False, help='Skip login and launch game without online authentication') - launch_parser.add_argument('--skip-version-check', dest='skip_version_check', action='store_true', - default=False, help='Skip version check when launching game in online mode') - launch_parser.add_argument('--override-username', dest='user_name_override', action='store', metavar='', - help='Override username used when launching the game (only works with some titles)') - launch_parser.add_argument('--dry-run', dest='dry_run', action='store_true', - help='Print the command line that would have been used to launch the game and exit') - launch_parser.add_argument('--language', dest='language', action='store', metavar='', - help='Override language for game launch (defaults to system locale)') - launch_parser.add_argument('--wrapper', dest='wrapper', action='store', metavar='', - default=os.environ.get('LGDRY_WRAPPER', None), - help='Wrapper command to launch game with') - launch_parser.add_argument('--set-defaults', dest='set_defaults', action='store_true', - help='Save parameters used to launch to config (does not include env vars)') - launch_parser.add_argument('--reset-defaults', dest='reset_defaults', action='store_true', - help='Reset config settings for app and exit') - launch_parser.add_argument('--override-exe', dest='executable_override', action='store', metavar='', - help='Override executable to launch (relative path)') - - if os.name != 'nt': - launch_parser.add_argument('--wine', dest='wine_bin', action='store', metavar='', - default=os.environ.get('LGDRY_WINE_BINARY', None), - help='Set WINE binary to use to launch the app') - launch_parser.add_argument('--wine-prefix', dest='wine_pfx', action='store', metavar='', - default=os.environ.get('LGDRY_WINE_PREFIX', None), - help='Set WINE prefix to use') - launch_parser.add_argument('--no-wine', dest='no_wine', action='store_true', - default=strtobool(os.environ.get('LGDRY_NO_WINE', 'False')), - help='Do not run game with WINE (e.g. if a wrapper is used)') - else: - # hidden arguments to not break this on Windows - launch_parser.add_argument('--wine', help=argparse.SUPPRESS, dest='wine_bin') - launch_parser.add_argument('--wine-prefix', help=argparse.SUPPRESS, dest='wine_pfx') - launch_parser.add_argument('--no-wine', dest='no_wine', help=argparse.SUPPRESS, - action='store_true', default=True) - - list_parser.add_argument('--platform', dest='platform_override', action='store', metavar='', - type=str, help='Override platform that games are shown for (e.g. Win32/Mac)') - list_parser.add_argument('--include-ue', dest='include_ue', action='store_true', - help='Also include Unreal Engine content (Engine/Marketplace) in list') - list_parser.add_argument('--csv', dest='csv', action='store_true', help='List games in CSV format') - list_parser.add_argument('--tsv', dest='tsv', action='store_true', help='List games in TSV format') - list_parser.add_argument('--json', dest='json', action='store_true', help='List games in JSON format') - - list_installed_parser.add_argument('--check-updates', dest='check_updates', action='store_true', - help='Check for updates for installed games') - list_installed_parser.add_argument('--csv', dest='csv', action='store_true', - help='List games in CSV format') - list_installed_parser.add_argument('--tsv', dest='tsv', action='store_true', - help='List games in TSV format') - list_installed_parser.add_argument('--json', dest='json', action='store_true', - help='List games in JSON format') - list_installed_parser.add_argument('--show-dirs', dest='include_dir', action='store_true', - help='Print installation directory in output') - - list_files_parser.add_argument('--force-download', dest='force_download', action='store_true', - help='Always download instead of using on-disk manifest') - list_files_parser.add_argument('--platform', dest='platform_override', action='store', metavar='', - type=str, help='Platform override for download (disables install)') - list_files_parser.add_argument('--manifest', dest='override_manifest', action='store', metavar='', - help='Manifest URL or path to use instead of the CDN one') - list_files_parser.add_argument('--csv', dest='csv', action='store_true', help='Output in CSV format') - list_files_parser.add_argument('--tsv', dest='tsv', action='store_true', help='Output in TSV format') - list_files_parser.add_argument('--json', dest='json', action='store_true', help='Output in JSON format') - list_files_parser.add_argument('--hashlist', dest='hashlist', action='store_true', - help='Output file hash list in hashcheck/sha1sum -c compatible format') - list_files_parser.add_argument('--install-tag', dest='install_tag', action='store', metavar='', - type=str, help='Show only files with specified install tag') - - sync_saves_parser.add_argument('--skip-upload', dest='download_only', action='store_true', - help='Only download new saves from cloud, don\'t upload') - sync_saves_parser.add_argument('--skip-download', dest='upload_only', action='store_true', - help='Only upload new saves from cloud, don\'t download') - sync_saves_parser.add_argument('--force-upload', dest='force_upload', action='store_true', - help='Force upload even if local saves are older') - sync_saves_parser.add_argument('--force-download', dest='force_download', action='store_true', - help='Force download even if local saves are newer') - sync_saves_parser.add_argument('--save-path', dest='save_path', action='store', metavar='', - help='Override savegame path (requires single app name to be specified)') - sync_saves_parser.add_argument('--disable-filters', dest='disable_filters', action='store_true', - help='Disable save game file filtering') - - import_parser.add_argument('--disable-check', dest='disable_check', action='store_true', - help='Disables completeness check of the to-be-imported game installation ' - '(useful if the imported game is a much older version or missing files)') - - egl_sync_parser.add_argument('--egl-manifest-path', dest='egl_manifest_path', action='store', - help='Path to the Epic Games Launcher\'s Manifests folder, should ' - 'point to /ProgramData/Epic/EpicGamesLauncher/Data/Manifests') - egl_sync_parser.add_argument('--egl-wine-prefix', dest='egl_wine_prefix', action='store', - help='Path to the WINE prefix the Epic Games Launcher is installed in') - egl_sync_parser.add_argument('--enable-sync', dest='enable_sync', action='store_true', - help='Enable automatic EGL <-> Legendary sync') - egl_sync_parser.add_argument('--disable-sync', dest='disable_sync', action='store_true', - help='Disable automatic sync and exit') - egl_sync_parser.add_argument('--one-shot', dest='one_shot', action='store_true', - help='Sync once, do not ask to setup automatic sync') - egl_sync_parser.add_argument('--import-only', dest='import_only', action='store_true', - help='Only import games from EGL (no export)') - egl_sync_parser.add_argument('--export-only', dest='export_only', action='store_true', - help='Only export games to EGL (no import)') - egl_sync_parser.add_argument('--unlink', dest='unlink', action='store_true', - help='Disable sync and remove EGL metadata from installed games') - - status_parser.add_argument('--offline', dest='offline', action='store_true', - help='Only print offline status information, do not login') - status_parser.add_argument('--json', dest='json', action='store_true', - help='Show status in JSON format') - - clean_parser.add_argument('--keep-manifests', dest='keep_manifests', action='store_true', - help='Do not delete old manifests') - - args, extra = parser.parse_known_args() - - if args.version: - print(f'legendary version "{__version__}", codename "{__codename__}"') - exit(0) - - if args.subparser_name not in ('auth', 'list-games', 'list-installed', 'list-files', - 'launch', 'download', 'uninstall', 'install', 'update', - 'repair', 'list-saves', 'download-saves', 'sync-saves', - 'verify-game', 'import-game', 'egl-sync', 'status', 'cleanup'): - print(parser.format_help()) - - # Print the main help *and* the help for all of the subcommands. Thanks stackoverflow! - print('Individual command help:') - subparsers = next(a for a in parser._actions if isinstance(a, argparse._SubParsersAction)) - for choice, subparser in subparsers.choices.items(): - if choice in ('download', 'update', 'repair'): - continue - print(f'\nCommand: {choice}') - print(subparser.format_help()) - return - - cli = LegendaryCLI() - ql = cli.setup_threaded_logging() - - config_ll = cli.core.lgd.config.get('Legendary', 'log_level', fallback='info') - if config_ll == 'debug' or args.debug: - logging.getLogger().setLevel(level=logging.DEBUG) - # keep requests quiet - logging.getLogger('requests').setLevel(logging.WARNING) - logging.getLogger('urllib3').setLevel(logging.WARNING) - - # if --yes is used as part of the subparsers arguments manually set the flag in the main parser. - if '-y' in extra or '--yes' in extra: - args.yes = True - extra = [i for i in extra if i not in ('--yes', '-y')] - - # technically args.func() with setdefaults could work (see docs on subparsers) - # but that would require all funcs to accept args and extra... - try: - if args.subparser_name == 'auth': - cli.auth(args) - elif args.subparser_name == 'list-games': - cli.list_games(args) - elif args.subparser_name == 'list-installed': - cli.list_installed(args) - elif args.subparser_name == 'launch': - cli.launch_game(args, extra) - elif args.subparser_name in ('download', 'install', 'update', 'repair'): - cli.install_game(args) - elif args.subparser_name == 'uninstall': - cli.uninstall_game(args) - elif args.subparser_name == 'list-files': - cli.list_files(args) - elif args.subparser_name == 'list-saves': - cli.list_saves(args) - elif args.subparser_name == 'download-saves': - cli.download_saves(args) - elif args.subparser_name == 'sync-saves': - cli.sync_saves(args) - elif args.subparser_name == 'verify-game': - cli.verify_game(args) - elif args.subparser_name == 'import-game': - cli.import_game(args) - elif args.subparser_name == 'egl-sync': - cli.egs_sync(args) - elif args.subparser_name == 'status': - cli.status(args) - elif args.subparser_name == 'cleanup': - cli.cleanup(args) - except KeyboardInterrupt: - logger.info('Command was aborted via KeyboardInterrupt, cleaning up...') - - cli.core.exit() - ql.stop() - exit(0) - - -if __name__ == '__main__': - # required for pyinstaller on Windows, does nothing on other platforms. - freeze_support() - main() diff --git a/custom_legendary/core.py b/custom_legendary/core.py deleted file mode 100644 index c5a6efcb..00000000 --- a/custom_legendary/core.py +++ /dev/null @@ -1,1417 +0,0 @@ -# coding: utf-8 - -import json -import logging -import os -import shlex -import shutil -from base64 import b64decode -from collections import defaultdict -from datetime import datetime, timezone -from locale import getdefaultlocale -from multiprocessing import Queue -from random import choice as randchoice -from typing import List, Dict, Callable -from uuid import uuid4 - -from requests import session -from requests.exceptions import HTTPError - -from custom_legendary.api.egs import EPCAPI -from custom_legendary.downloader.manager import DLManager -from custom_legendary.lfs.egl import EPCLFS -from custom_legendary.lfs.lgndry import LGDLFS -from custom_legendary.models.chunk import Chunk -from custom_legendary.models.downloading import AnalysisResult, ConditionCheckResult -from custom_legendary.models.egl import EGLManifest -from custom_legendary.models.exceptions import InvalidCredentialsError -from custom_legendary.models.game import GameAsset, Game, InstalledGame, SaveGameFile, SaveGameStatus, VerifyResult -from custom_legendary.models.json_manifest import JSONManifest -from custom_legendary.models.manifest import Manifest -from custom_legendary.utils.game_workarounds import is_opt_enabled -from custom_legendary.utils.lfs import clean_filename, delete_folder, delete_filelist, validate_files -from custom_legendary.utils.manifests import combine_manifests -from custom_legendary.utils.savegame_helper import SaveGameHelper -from custom_legendary.utils.selective_dl import get_sdl_appname -from custom_legendary.utils.wine_helpers import read_registry, get_shell_folders - - -# ToDo: instead of true/false return values for success/failure actually raise an exception that the CLI/GUI -# can handle to give the user more details. (Not required yet since there's no GUI so log output is fine) - - -class LegendaryCore: - """ - LegendaryCore handles most of the lower level interaction with - the downloader, lfs, and api components to make writing CLI/GUI - code easier and cleaner and avoid duplication. - """ - - def __init__(self): - self.log = logging.getLogger('Core') - self.egs = EPCAPI() - self.lgd = LGDLFS() - self.egl = EPCLFS() - - # on non-Windows load the programdata path from config - if os.name != 'nt': - self.egl.programdata_path = self.lgd.config.get('Legendary', 'egl_programdata', fallback=None) - if self.egl.programdata_path and not os.path.exists(self.egl.programdata_path): - self.log.error(f'Config EGL path ("{self.egl.programdata_path}") is invalid! Disabling sync...') - self.egl.programdata_path = None - self.lgd.config.remove_option('Legendary', 'egl_programdata') - self.lgd.config.remove_option('Legendary', 'egl_sync') - self.lgd.save_config() - - self.local_timezone = datetime.now().astimezone().tzinfo - self.language_code, self.country_code = ('en', 'US') - - if locale := self.lgd.config.get('Legendary', 'locale', fallback=getdefaultlocale()[0]): - try: - self.language_code, self.country_code = locale.split('-' if '-' in locale else '_') - self.log.debug(f'Set locale to {self.language_code}-{self.country_code}') - # adjust egs api language as well - self.egs.language_code, self.egs.country_code = self.language_code, self.country_code - except Exception as e: - self.log.warning(f'Getting locale failed: {e!r}, falling back to using en-US.') - else: - self.log.warning(f'Could not determine locale, falling back to en-US') - - def auth(self, username, password): - """ - Attempts direct non-web login, raises CaptchaError if manual login is required - - :param username: - :param password: - :return: - """ - raise NotImplementedError - - def auth_sid(self, sid) -> str: - """ - Handles getting an exchange code from a session id - :param sid: session id - :return: exchange code - """ - s = session() - s.headers.update({ - 'X-Epic-Event-Action': 'login', - 'X-Epic-Event-Category': 'login', - 'X-Epic-Strategy-Flags': '', - 'X-Requested-With': 'XMLHttpRequest', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' - 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'EpicGamesLauncher/11.0.1-14907503+++Portal+Release-Live ' - 'UnrealEngine/4.23.0-14907503+++Portal+Release-Live ' - 'Chrome/84.0.4147.38 Safari/537.36' - }) - - # get first set of cookies (EPIC_BEARER_TOKEN etc.) - _ = s.get('https://www.epicgames.com/id/api/set-sid', params=dict(sid=sid)) - # get XSRF-TOKEN and EPIC_SESSION_AP cookie - _ = s.get('https://www.epicgames.com/id/api/csrf') - # finally, get the exchange code - r = s.post('https://www.epicgames.com/id/api/exchange/generate', - headers={'X-XSRF-TOKEN': s.cookies['XSRF-TOKEN']}) - - if r.status_code == 200: - return r.json()['code'] - else: - self.log.error(f'Getting exchange code failed: {r.json()}') - return '' - - def auth_code(self, code) -> bool: - """ - Handles authentication via exchange code (either retrieved manually or automatically) - """ - try: - self.lgd.userdata = self.egs.start_session(exchange_token=code) - return True - except Exception as e: - self.log.error(f'Logging in failed with {e!r}, please try again.') - return False - - def auth_import(self) -> bool: - """Import refresh token from EGL installation and use it for logging in""" - self.egl.read_config() - remember_me_data = self.egl.config.get('RememberMe', 'Data') - re_data = json.loads(b64decode(remember_me_data))[0] - if 'Token' not in re_data: - raise ValueError('No login session in config') - refresh_token = re_data['Token'] - try: - self.lgd.userdata = self.egs.start_session(refresh_token=refresh_token) - return True - except Exception as e: - self.log.error(f'Logging in failed with {e!r}, please try again.') - return False - - def login(self) -> bool: - """ - Attempts logging in with existing credentials. - - raises ValueError if no existing credentials or InvalidCredentialsError if the API return an error - """ - if not self.lgd.userdata: - raise ValueError('No saved credentials') - - if self.lgd.userdata['expires_at']: - dt_exp = datetime.fromisoformat(self.lgd.userdata['expires_at'][:-1]) - dt_now = datetime.utcnow() - td = dt_now - dt_exp - - # if session still has at least 10 minutes left we can re-use it. - if dt_exp > dt_now and abs(td.total_seconds()) > 600: - self.log.info('Trying to re-use existing login session...') - try: - self.egs.resume_session(self.lgd.userdata) - return True - except InvalidCredentialsError as e: - self.log.warning(f'Resuming failed due to invalid credentials: {e!r}') - except Exception as e: - self.log.warning(f'Resuming failed for unknown reason: {e!r}') - # If verify fails just continue the normal authentication process - self.log.info('Falling back to using refresh token...') - - try: - self.log.info('Logging in...') - userdata = self.egs.start_session(self.lgd.userdata['refresh_token']) - except InvalidCredentialsError: - self.log.error('Stored credentials are no longer valid! Please login again.') - self.lgd.invalidate_userdata() - return False - except HTTPError as e: - self.log.error(f'HTTP request for login failed: {e!r}, please try again later.') - return False - - self.lgd.userdata = userdata - return True - - def get_assets(self, update_assets=False, platform_override=None) -> List[GameAsset]: - # do not save and always fetch list when platform is overridden - if platform_override: - return [GameAsset.from_egs_json(a) for a in - self.egs.get_game_assets(platform=platform_override)] - - if not self.lgd.assets or update_assets: - # if not logged in, return empty list - if not self.egs.user: - return [] - self.lgd.assets = [GameAsset.from_egs_json(a) for a in self.egs.get_game_assets()] - - return self.lgd.assets - - def get_asset(self, app_name, update=False) -> GameAsset: - if update: - self.get_assets(update_assets=True) - - try: - return next(i for i in self.lgd.assets if i.app_name == app_name) - except StopIteration: - raise ValueError - - def asset_valid(self, app_name) -> bool: - return any(i.app_name == app_name for i in self.lgd.assets) - - def get_game(self, app_name, update_meta=False) -> Game: - if update_meta: - self.get_game_list(True) - return self.lgd.get_game_meta(app_name) - - def get_game_list(self, update_assets=True) -> List[Game]: - return self.get_game_and_dlc_list(update_assets=update_assets)[0] - - def get_game_and_dlc_list(self, update_assets=True, - platform_override=None, - skip_ue=True) -> (List[Game], Dict[str, Game]): - _ret = [] - _dlc = defaultdict(list) - - for ga in self.get_assets(update_assets=update_assets, - platform_override=platform_override): - if ga.namespace == 'ue' and skip_ue: - continue - - game = self.lgd.get_game_meta(ga.app_name) - if update_assets and (not game or - (game and game.app_version != ga.build_version and not platform_override)): - if game and game.app_version != ga.build_version and not platform_override: - self.log.info(f'Updating meta for {game.app_name} due to build version mismatch') - - eg_meta = self.egs.get_game_info(ga.namespace, ga.catalog_item_id) - game = Game(app_name=ga.app_name, app_version=ga.build_version, - app_title=eg_meta['title'], asset_info=ga, metadata=eg_meta) - - if not platform_override: - self.lgd.set_game_meta(game.app_name, game) - - # replace asset info with the platform specific one if override is used - if platform_override: - game.app_version = ga.build_version - game.asset_info = ga - - if game.is_dlc: - _dlc[game.metadata['mainGameItem']['id']].append(game) - elif not any(i['path'] == 'mods' for i in game.metadata.get('categories', [])): - _ret.append(game) - - return _ret, _dlc - - def get_dlc_for_game(self, app_name): - game = self.get_game(app_name) - if not game: - self.log.warning(f'Metadata for {app_name} is missing!') - return [] - - if game.is_dlc: # dlc shouldn't have DLC - return [] - - _, dlcs = self.get_game_and_dlc_list(update_assets=False) - return dlcs[game.asset_info.catalog_item_id] - - def get_installed_list(self) -> List[InstalledGame]: - if self.egl_sync_enabled: - self.log.debug('Running EGL sync...') - self.egl_sync() - - return self._get_installed_list() - - def _get_installed_list(self) -> List[InstalledGame]: - return [g for g in self.lgd.get_installed_list() if not g.is_dlc] - - def get_installed_dlc_list(self) -> List[InstalledGame]: - return [g for g in self.lgd.get_installed_list() if g.is_dlc] - - def get_installed_game(self, app_name) -> InstalledGame: - igame = self._get_installed_game(app_name) - if igame and self.egl_sync_enabled and igame.egl_guid and not igame.is_dlc: - self.egl_sync(app_name) - return self._get_installed_game(app_name) - else: - return igame - - def _get_installed_game(self, app_name) -> InstalledGame: - return self.lgd.get_installed_game(app_name) - - def get_non_asset_library_items(self, skip_ue=True) -> (List[Game], Dict[str, List[Game]]): - """ - Gets a list of Games without assets for installation, for instance Games delivered via - third-party stores that do not have assets for installation - :param skip_ue: Ingore Unreal Marketplace entries - :return: List of Games and DLC that do not have assets - """ - _ret = [] - _dlc = defaultdict(list) - # get all the appnames we have to ignore - ignore = set(i.app_name for i in self.get_assets()) - - for libitem in self.egs.get_library_items(): - if libitem['namespace'] == 'ue' and skip_ue: - continue - if libitem['appName'] in ignore: - continue - - game = self.lgd.get_game_meta(libitem['appName']) - if not game: - eg_meta = self.egs.get_game_info(libitem['namespace'], libitem['catalogItemId']) - game = Game(app_name=libitem['appName'], app_version=None, - app_title=eg_meta['title'], asset_info=None, metadata=eg_meta) - self.lgd.set_game_meta(game.app_name, game) - if game.is_dlc: - _dlc[game.metadata['mainGameItem']['id']].append(game) - elif not any(i['path'] == 'mods' for i in game.metadata.get('categories', [])): - _ret.append(game) - - return _ret, _dlc - - def get_launch_parameters(self, app_name: str, offline: bool = False, - user: str = None, extra_args: list = None, - wine_bin: str = None, wine_pfx: str = None, - language: str = None, wrapper: str = None, - disable_wine: bool = False, - executable_override: str = None) -> (list, str, dict): - install = self.lgd.get_installed_game(app_name) - game = self.lgd.get_game_meta(app_name) - - game_token = '' - if not offline: - self.log.info('Getting authentication token...') - game_token = self.egs.get_game_token()['code'] - elif not install.can_run_offline: - self.log.warning('Game is not approved for offline use and may not work correctly.') - - user_name = self.lgd.userdata['displayName'] - account_id = self.lgd.userdata['account_id'] - if user: - user_name = user - - if executable_override or (executable_override := self.lgd.config.get(app_name, 'override_exe', fallback=None)): - game_exe = os.path.join(install.install_path, - executable_override.replace('\\', '/')) - if not os.path.exists(game_exe): - raise ValueError(f'Executable path is invalid: {game_exe}') - else: - game_exe = os.path.join(install.install_path, - install.executable.replace('\\', '/').lstrip('/')) - - working_dir = os.path.split(game_exe)[0] - - params = [] - - if wrapper or (wrapper := self.lgd.config.get(app_name, 'wrapper', - fallback=self.lgd.config.get('default', 'wrapper', - fallback=None))): - params.extend(shlex.split(wrapper)) - - if os.name != 'nt' and not disable_wine: - if not wine_bin: - # check if there's a default override - wine_bin = self.lgd.config.get('default', 'wine_executable', fallback='wine') - # check if there's a game specific override - wine_bin = self.lgd.config.get(app_name, 'wine_executable', fallback=wine_bin) - - if not self.lgd.config.getboolean(app_name, 'no_wine', - fallback=self.lgd.config.get('default', 'no_wine', fallback=False)): - params.append(wine_bin) - - params.append(game_exe) - - if install.launch_parameters: - params.extend(shlex.split(install.launch_parameters, posix=False)) - - params.extend([ - '-AUTH_LOGIN=unused', - f'-AUTH_PASSWORD={game_token}', - '-AUTH_TYPE=exchangecode', - f'-epicapp={app_name}', - '-epicenv=Prod']) - - if install.requires_ot and not offline: - self.log.info('Getting ownership token.') - ovt = self.egs.get_ownership_token(game.asset_info.namespace, - game.asset_info.catalog_item_id) - ovt_path = os.path.join(self.lgd.get_tmp_path(), - f'{game.asset_info.namespace}{game.asset_info.catalog_item_id}.ovt') - with open(ovt_path, 'wb') as f: - f.write(ovt) - params.append(f'-epicovt={ovt_path}') - - language_code = self.lgd.config.get(app_name, 'language', fallback=language) - if not language_code: # fall back to system or config language - language_code = self.language_code - - params.extend([ - '-EpicPortal', - f'-epicusername={user_name}', - f'-epicuserid={account_id}', - f'-epiclocale={language_code}' - ]) - - if extra_args: - params.extend(extra_args) - - if config_args := self.lgd.config.get(app_name, 'start_params', fallback=None): - params.extend(shlex.split(config_args.strip())) - - # get environment overrides from config - env = os.environ.copy() - if 'default.env' in self.lgd.config: - env.update({k: v for k, v in self.lgd.config[f'default.env'].items() if v and not k.startswith(';')}) - if f'{app_name}.env' in self.lgd.config: - env.update({k: v for k, v in self.lgd.config[f'{app_name}.env'].items() if v and not k.startswith(';')}) - - if wine_pfx: - env['WINEPREFIX'] = wine_pfx - elif 'WINEPREFIX' not in env: - # only use config variable if not already set in environment - if wine_pfx := self.lgd.config.get(app_name, 'wine_prefix', fallback=None): - env['WINEPREFIX'] = wine_pfx - - return params, working_dir, env - - def get_save_games(self, app_name: str = ''): - savegames = self.egs.get_user_cloud_saves(app_name, manifests=not not app_name) - _saves = [] - for fname, f in savegames['files'].items(): - if '.manifest' not in fname: - continue - f_parts = fname.split('/') - _saves.append(SaveGameFile(app_name=f_parts[2], filename=fname, manifest=f_parts[4], - datetime=datetime.fromisoformat(f['lastModified'][:-1]))) - - return _saves - - def get_save_path(self, app_name): - game = self.lgd.get_game_meta(app_name) - save_path = game.metadata['customAttributes'].get('CloudSaveFolder', {}).get('value') - if not save_path: - raise ValueError('Game does not support cloud saves') - - igame = self.lgd.get_installed_game(app_name) - if not igame: - raise ValueError('Game is not installed!') - - # the following variables are known: - path_vars = { - '{installdir}': igame.install_path, - '{epicid}': self.lgd.userdata['account_id'] - } - - if os.name == 'nt': - path_vars.update({ - '{appdata}': os.path.expandvars('%APPDATA%'), - '{userdir}': os.path.expandvars('%userprofile%/documents'), - # '{userprofile}': os.path.expandvars('%userprofile%'), # possibly wrong - '{usersavedgames}': os.path.expandvars('%userprofile%/Saved Games') - }) - else: - # attempt to get WINE prefix from config - wine_pfx = self.lgd.config.get(app_name, 'wine_prefix', fallback=None) - if not wine_pfx: - wine_pfx = self.lgd.config.get(f'{app_name}.env', 'WINEPREFIX', fallback=None) - if not wine_pfx: - proton_pfx = self.lgd.config.get(f'{app_name}.env', 'STEAM_COMPAT_DATA_PATH', fallback=None) - if proton_pfx: - wine_pfx = f'{proton_pfx}/pfx' - if not wine_pfx: - wine_pfx = os.path.expanduser('~/.wine') - - # if we have a prefix, read the `user.reg` file and get the proper paths. - if os.path.isdir(wine_pfx): - wine_reg = read_registry(wine_pfx) - wine_folders = get_shell_folders(wine_reg, wine_pfx) - # path_vars['{userprofile}'] = user_path - path_vars['{appdata}'] = wine_folders['AppData'] - # this maps to ~/Documents, but the name is locale-dependent so just resolve the symlink from WINE - path_vars['{userdir}'] = os.path.realpath(wine_folders['Personal']) - path_vars['{usersavedgames}'] = wine_folders['{4C5C32FF-BB9D-43B0-B5B4-2D72E54EAAA4}'] - - # replace backslashes - save_path = save_path.replace('\\', '/') - - # these paths should always use a forward slash - new_save_path = [path_vars.get(p.lower(), p) for p in save_path.split('/')] - return os.path.realpath(os.path.join(*new_save_path)) - - def check_savegame_state(self, path: str, save: SaveGameFile) -> (SaveGameStatus, (datetime, datetime)): - latest = 0 - for _dir, _, _files in os.walk(path): - for _file in _files: - s = os.stat(os.path.join(_dir, _file)) - latest = max(latest, s.st_mtime) - - if not latest and not save: - return SaveGameStatus.NO_SAVE, (None, None) - - # timezones are fun! - dt_local = datetime.fromtimestamp(latest).replace(tzinfo=self.local_timezone).astimezone(timezone.utc) - if not save: - return SaveGameStatus.LOCAL_NEWER, (dt_local, None) - - dt_remote = datetime.strptime(save.manifest_name, '%Y.%m.%d-%H.%M.%S.manifest').replace(tzinfo=timezone.utc) - if not latest: - return SaveGameStatus.REMOTE_NEWER, (None, dt_remote) - - self.log.debug(f'Local save date: {str(dt_local)}, Remote save date: {str(dt_remote)}') - - # Ideally we check the files themselves based on manifest, - # this is mostly a guess but should be accurate enough. - if abs((dt_local - dt_remote).total_seconds()) < 60: - return SaveGameStatus.SAME_AGE, (dt_local, dt_remote) - elif dt_local > dt_remote: - return SaveGameStatus.LOCAL_NEWER, (dt_local, dt_remote) - else: - return SaveGameStatus.REMOTE_NEWER, (dt_local, dt_remote) - - def upload_save(self, app_name, save_dir, local_dt: datetime = None, - disable_filtering: bool = False): - game = self.lgd.get_game_meta(app_name) - custom_attr = game.metadata['customAttributes'] - save_path = custom_attr.get('CloudSaveFolder', {}).get('value') - - include_f = exclude_f = None - if not disable_filtering: - # get file inclusion and exclusion filters if they exist - if (_include := custom_attr.get('CloudIncludeList', {}).get('value', None)) is not None: - include_f = _include.split(',') - if (_exclude := custom_attr.get('CloudExcludeList', {}).get('value', None)) is not None: - exclude_f = _exclude.split(',') - - if not save_path: - raise ValueError('Game does not support cloud saves') - - sgh = SaveGameHelper() - files = sgh.package_savegame(save_dir, app_name, self.egs.user.get('account_id'), - save_path, include_f, exclude_f, local_dt) - - if not files: - self.log.info('No files to upload. If you believe this is incorrect run command with "--disable-filters"') - return - - self.log.debug(f'Packed files: {str(files)}, creating cloud files...') - resp = self.egs.create_game_cloud_saves(app_name, list(files.keys())) - - self.log.info('Starting upload...') - for remote_path, file_info in resp['files'].items(): - self.log.debug(f'Uploading "{remote_path}"') - f = files.get(remote_path) - self.egs.unauth_session.put(file_info['writeLink'], data=f.read()) - - self.log.info('Finished uploading savegame.') - - def download_saves(self, app_name='', manifest_name='', save_dir='', clean_dir=False): - save_path = os.path.join(self.get_default_install_dir(), '.saves') - if not os.path.exists(save_path): - os.makedirs(save_path) - - _save_dir = save_dir - savegames = self.egs.get_user_cloud_saves(app_name=app_name) - files = savegames['files'] - for fname, f in files.items(): - if '.manifest' not in fname: - continue - f_parts = fname.split('/') - - if manifest_name and f_parts[4] != manifest_name: - continue - if not save_dir: - _save_dir = os.path.join(save_path, f'{f_parts[2]}/{f_parts[4].rpartition(".")[0]}') - if not os.path.exists(_save_dir): - os.makedirs(_save_dir) - - if clean_dir: - self.log.info('Deleting old save files...') - delete_folder(_save_dir) - - self.log.info(f'Downloading "{fname.split("/", 2)[2]}"...') - # download manifest - r = self.egs.unauth_session.get(f['readLink']) - if r.status_code != 200: - self.log.error(f'Download failed, status code: {r.status_code}') - continue - - if not r.content: - self.log.error('Manifest is empty! Skipping...') - continue - - m = self.load_manifest(r.content) - - # download chunks required for extraction - chunks = dict() - for chunk in m.chunk_data_list.elements: - cpath_p = fname.split('/', 3)[:3] - cpath_p.append(chunk.path) - cpath = '/'.join(cpath_p) - self.log.debug(f'Downloading chunk "{cpath}"') - r = self.egs.unauth_session.get(files[cpath]['readLink']) - if r.status_code != 200: - self.log.error(f'Download failed, status code: {r.status_code}') - break - c = Chunk.read_buffer(r.content) - chunks[c.guid_num] = c.data - - for fm in m.file_manifest_list.elements: - dirs, fname = os.path.split(fm.filename) - fdir = os.path.join(_save_dir, dirs) - fpath = os.path.join(fdir, fname) - if not os.path.exists(fdir): - os.makedirs(fdir) - - self.log.debug(f'Writing "{fpath}"...') - with open(fpath, 'wb') as fh: - for cp in fm.chunk_parts: - fh.write(chunks[cp.guid_num][cp.offset:cp.offset + cp.size]) - - # set modified time to savegame creation timestamp - m_date = datetime.strptime(f_parts[4], '%Y.%m.%d-%H.%M.%S.manifest') - m_date = m_date.replace(tzinfo=timezone.utc).astimezone(self.local_timezone) - os.utime(fpath, (m_date.timestamp(), m_date.timestamp())) - - self.log.info('Successfully completed savegame download.') - - def is_offline_game(self, app_name: str) -> bool: - return self.lgd.config.getboolean(app_name, 'offline', fallback=False) - - def is_noupdate_game(self, app_name: str) -> bool: - return self.lgd.config.getboolean(app_name, 'skip_update_check', fallback=False) - - def is_latest(self, app_name: str) -> bool: - installed = self.lgd.get_installed_game(app_name) - - for ass in self.get_assets(True): - if ass.app_name == app_name: - if ass.build_version != installed.version: - return False - else: - return True - # if we get here something is very wrong - raise ValueError(f'Could not find {app_name} in asset list!') - - def is_installed(self, app_name: str) -> bool: - return self.get_installed_game(app_name) is not None - - def _is_installed(self, app_name: str) -> bool: - return self._get_installed_game(app_name) is not None - - def is_dlc(self, app_name: str) -> bool: - meta = self.lgd.get_game_meta(app_name) - if not meta: - raise ValueError('Game unknown!') - return meta.is_dlc - - @staticmethod - def load_manifest(data: bytes) -> Manifest: - if data[0:1] == b'{': - return JSONManifest.read_all(data) - else: - return Manifest.read_all(data) - - def get_installed_manifest(self, app_name): - igame = self._get_installed_game(app_name) - old_bytes = self.lgd.load_manifest(app_name, igame.version) - return old_bytes, igame.base_urls - - def get_cdn_urls(self, game, platform_override=''): - platform = 'Windows' if not platform_override else platform_override - m_api_r = self.egs.get_game_manifest(game.asset_info.namespace, - game.asset_info.catalog_item_id, - game.app_name, platform) - - # never seen this outside the launcher itself, but if it happens: PANIC! - if len(m_api_r['elements']) > 1: - raise ValueError('Manifest response has more than one element!') - - base_urls = [] - manifest_urls = [] - for manifest in m_api_r['elements'][0]['manifests']: - base_url = manifest['uri'].rpartition('/')[0] - if base_url not in base_urls: - base_urls.append(base_url) - - if 'queryParams' in manifest: - params = '&'.join(f'{p["name"]}={p["value"]}' for p in manifest['queryParams']) - manifest_urls.append(f'{manifest["uri"]}?{params}') - else: - manifest_urls.append(manifest['uri']) - - return manifest_urls, base_urls - - def get_cdn_manifest(self, game, platform_override=''): - manifest_urls, base_urls = self.get_cdn_urls(game, platform_override) - self.log.debug(f'Downloading manifest from {manifest_urls[0]} ...') - r = self.egs.unauth_session.get(manifest_urls[0]) - r.raise_for_status() - return r.content, base_urls - - def get_uri_manifest(self, uri): - if uri.startswith('http'): - r = self.egs.unauth_session.get(uri) - r.raise_for_status() - new_manifest_data = r.content - base_urls = [r.url.rpartition('/')[0]] - else: - base_urls = [] - with open(uri, 'rb') as f: - new_manifest_data = f.read() - - return new_manifest_data, base_urls - - def get_delta_manifest(self, base_url, old_build_id, new_build_id): - """Get optimized delta manifest (doesn't seem to exist for most games)""" - if old_build_id == new_build_id: - return None - - r = self.egs.unauth_session.get(f'{base_url}/Deltas/{new_build_id}/{old_build_id}.delta') - if r.status_code == 200: - return r.content - else: - return None - - def verify_game(self, app_name: str, callback: Callable[[int, int], None] = print): - if not self.is_installed(app_name): - self.log.error(f'Game "{app_name}" is not installed') - return - - self.log.info(f'Loading installed manifest for "{app_name}"') - igame = self.get_installed_game(app_name) - manifest_data, _ = self.get_installed_manifest(app_name) - manifest = self.load_manifest(manifest_data) - - files = sorted(manifest.file_manifest_list.elements, - key=lambda a: a.filename.lower()) - - # build list of hashes - file_list = [(f.filename, f.sha_hash.hex()) for f in files] - total = len(file_list) - num = 0 - failed = [] - missing = [] - - self.log.info(f'Verifying "{igame.title}" version "{manifest.meta.build_version}"') - repair_file = [] - for result, path, result_hash in validate_files(igame.install_path, file_list): - if callback: - num += 1 - callback(num, total) - - if result == VerifyResult.HASH_MATCH: - repair_file.append(f'{result_hash}:{path}') - continue - elif result == VerifyResult.HASH_MISMATCH: - self.log.error(f'File does not match hash: "{path}"') - repair_file.append(f'{result_hash}:{path}') - failed.append(path) - elif result == VerifyResult.FILE_MISSING: - self.log.error(f'File is missing: "{path}"') - missing.append(path) - else: - self.log.error(f'Other failure (see log), treating file as missing: "{path}"') - missing.append(path) - - # always write repair file, even if all match - if repair_file: - repair_filename = os.path.join(self.lgd.get_tmp_path(), f'{app_name}.repair') - with open(repair_filename, 'w') as f: - f.write('\n'.join(repair_file)) - self.log.debug(f'Written repair file to "{repair_filename}"') - - if not missing and not failed: - self.log.info('Verification finished successfully.') - else: - raise RuntimeError( - f'Verification failed, {len(failed)} file(s) corrupted, {len(missing)} file(s) are missing.') - - def prepare_download(self, app_name: str, base_path: str = '', no_install: bool = False, - status_q: Queue = None, max_shm: int = 0, max_workers: int = 0, - force: bool = False, disable_patching: bool = False, - game_folder: str = '', override_manifest: str = '', - override_old_manifest: str = '', override_base_url: str = '', - platform_override: str = '', file_prefix_filter: list = None, - file_exclude_filter: list = None, file_install_tag: list = None, - dl_optimizations: bool = False, dl_timeout: int = 10, - repair: bool = False, repair_use_latest: bool = False, - ignore_space_req: bool = False, - disable_delta: bool = False, override_delta_manifest: str = '', - egl_guid: str = '', reset_sdl: bool = False, - sdl_prompt: Callable[[str, str], List[str]] = list) -> ( - DLManager, AnalysisResult, Game, InstalledGame, bool, str): - if self.is_installed(app_name): - igame = self.get_installed_game(app_name) - if igame.needs_verification and not repair: - self.log.info('Game needs to be verified before updating, switching to repair mode...') - repair = True - - repair_file = '' - if repair: - repair = True - no_install = repair_use_latest is False - repair_file = os.path.join(self.lgd.get_tmp_path(), f'{app_name}.repair') - - if file_prefix_filter or file_exclude_filter or file_install_tag: - no_install = True - - if platform_override: - no_install = True - - game = self.get_game(app_name, update_meta=True) - - if not game: - raise RuntimeError(f'Could not find "{app_name}" in list of available games,' - f'did you type the name correctly?') - - if game.is_dlc: - self.log.info('Install candidate is DLC') - app_name = game.metadata['mainGameItem']['releaseInfo'][0]['appId'] - base_game = self.get_game(app_name) - # check if base_game is actually installed - if not self.is_installed(app_name): - # download mode doesn't care about whether or not something's installed - if not no_install: - raise RuntimeError(f'Base game "{app_name}" is not installed!') - else: - base_game = None - - if repair: - if not self.is_installed(game.app_name): - raise RuntimeError(f'Game "{game.app_title}" ({game.app_name}) is not installed!') - - if not os.path.exists(repair_file): - self.log.info('Verifing game...') - self.verify_game(app_name) - else: - self.log.info(f'Using existing repair file: {repair_file}') - - # Workaround for Cyberpunk 2077 preload - if not file_install_tag and not game.is_dlc and ((sdl_name := get_sdl_appname(game.app_name)) is not None): - config_tags = self.lgd.config.get(game.app_name, 'install_tags', fallback=None) - if not self.is_installed(game.app_name) or config_tags is None or reset_sdl: - file_install_tag = sdl_prompt(sdl_name, game.app_title) - if game.app_name not in self.lgd.config: - self.lgd.config[game.app_name] = dict() - self.lgd.config.set(game.app_name, 'install_tags', ','.join(file_install_tag)) - else: - file_install_tag = config_tags.split(',') - - # load old manifest - old_manifest = None - - # load old manifest if we have one - if override_old_manifest: - self.log.info(f'Overriding old manifest with "{override_old_manifest}"') - old_bytes, _ = self.get_uri_manifest(override_old_manifest) - old_manifest = self.load_manifest(old_bytes) - elif not disable_patching and not force and self.is_installed(game.app_name): - old_bytes, _base_urls = self.get_installed_manifest(game.app_name) - if _base_urls and not game.base_urls: - game.base_urls = _base_urls - - if not old_bytes: - self.log.error(f'Could not load old manifest, patching will not work!') - else: - old_manifest = self.load_manifest(old_bytes) - - base_urls = list(game.base_urls) # copy list for manipulation - - if override_manifest: - self.log.info(f'Overriding manifest with "{override_manifest}"') - new_manifest_data, _base_urls = self.get_uri_manifest(override_manifest) - # if override manifest has a base URL use that instead - if _base_urls: - base_urls = _base_urls - else: - new_manifest_data, _base_urls = self.get_cdn_manifest(game, platform_override) - base_urls.extend(i for i in _base_urls if i not in base_urls) - game.base_urls = base_urls - # save base urls to game metadata - self.lgd.set_game_meta(game.app_name, game) - - self.log.info('Parsing game manifest...') - new_manifest = self.load_manifest(new_manifest_data) - self.log.debug(f'Base urls: {base_urls}') - # save manifest with version name as well for testing/downgrading/etc. - self.lgd.save_manifest(game.app_name, new_manifest_data, - version=new_manifest.meta.build_version) - - # check if we should use a delta manifest or not - disable_delta = disable_delta or ((override_old_manifest or override_manifest) and not override_delta_manifest) - if old_manifest and new_manifest: - disable_delta = disable_delta or (old_manifest.meta.build_id == new_manifest.meta.build_id) - if old_manifest and new_manifest and not disable_delta: - if override_delta_manifest: - self.log.info(f'Overriding delta manifest with "{override_delta_manifest}"') - delta_manifest_data, _ = self.get_uri_manifest(override_delta_manifest) - else: - delta_manifest_data = self.get_delta_manifest(randchoice(base_urls), - old_manifest.meta.build_id, - new_manifest.meta.build_id) - if delta_manifest_data: - delta_manifest = self.load_manifest(delta_manifest_data) - self.log.info(f'Using optimized delta manifest to upgrade from build ' - f'"{old_manifest.meta.build_id}" to ' - f'"{new_manifest.meta.build_id}"...') - combine_manifests(new_manifest, delta_manifest) - else: - self.log.debug(f'No Delta manifest received from CDN.') - - # reuse existing installation's directory - if igame := self.get_installed_game(base_game.app_name if base_game else game.app_name): - install_path = igame.install_path - # make sure to re-use the epic guid we assigned on first install - if not game.is_dlc and igame.egl_guid: - egl_guid = igame.egl_guid - else: - if not game_folder: - if game.is_dlc: - game_folder = base_game.metadata.get('customAttributes', {}). \ - get('FolderName', {}).get('value', base_game.app_name) - else: - game_folder = game.metadata.get('customAttributes', {}). \ - get('FolderName', {}).get('value', game.app_name) - - if not base_path: - base_path = self.get_default_install_dir() - - # make sure base directory actually exists (but do not create game dir) - if not os.path.exists(base_path): - self.log.info(f'"{base_path}" does not exist, creating...') - os.makedirs(base_path) - - install_path = os.path.join(base_path, game_folder) - - self.log.info(f'Install path: {install_path}') - - if repair: - if not repair_use_latest: - # use installed manifest for repairs instead of updating - new_manifest = old_manifest - old_manifest = None - - filename = clean_filename(f'{game.app_name}.repair') - resume_file = os.path.join(self.lgd.get_tmp_path(), filename) - force = False - elif not force: - filename = clean_filename(f'{game.app_name}.resume') - resume_file = os.path.join(self.lgd.get_tmp_path(), filename) - else: - resume_file = None - - if override_base_url: - self.log.info(f'Overriding base URL with "{override_base_url}"') - base_url = override_base_url - else: - # randomly select one CDN - base_url = randchoice(base_urls) - - self.log.debug(f'Using base URL: {base_url}') - - if not max_shm: - max_shm = self.lgd.config.getint('Legendary', 'max_memory', fallback=2048) - - if dl_optimizations or is_opt_enabled(game.app_name, new_manifest.meta.build_version): - self.log.info('Download order optimizations are enabled.') - process_opt = True - else: - process_opt = False - - if not max_workers: - max_workers = self.lgd.config.getint('Legendary', 'max_workers', fallback=0) - - dlm = DLManager(install_path, base_url, resume_file=resume_file, status_q=status_q, - max_shared_memory=max_shm * 1024 * 1024, max_workers=max_workers, - dl_timeout=dl_timeout) - anlres = dlm.run_analysis(manifest=new_manifest, old_manifest=old_manifest, - patch=not disable_patching, resume=not force, - file_prefix_filter=file_prefix_filter, - file_exclude_filter=file_exclude_filter, - file_install_tag=file_install_tag, - processing_optimization=process_opt) - - prereq = None - if new_manifest.meta.prereq_ids: - prereq = dict(ids=new_manifest.meta.prereq_ids, name=new_manifest.meta.prereq_name, - path=new_manifest.meta.prereq_path, args=new_manifest.meta.prereq_args) - - offline = game.metadata.get('customAttributes', {}).get('CanRunOffline', {}).get('value', 'true') - ot = game.metadata.get('customAttributes', {}).get('OwnershipToken', {}).get('value', 'false') - - igame = InstalledGame(app_name=game.app_name, title=game.app_title, - version=new_manifest.meta.build_version, prereq_info=prereq, - manifest_path=override_manifest, base_urls=base_urls, - install_path=install_path, executable=new_manifest.meta.launch_exe, - launch_parameters=new_manifest.meta.launch_command, - can_run_offline=offline == 'true', requires_ot=ot == 'true', - is_dlc=base_game is not None, install_size=anlres.install_size, - egl_guid=egl_guid, install_tags=file_install_tag) - - # game is either up to date or hasn't changed, so we have nothing to do - if not anlres.dl_size: - old_igame = self.get_installed_game(game.app_name) - self.log.info('Download size is 0, the game is either already up to date or has not changed. Exiting...') - if old_igame and repair and os.path.exists(repair_file): - if old_igame.needs_verification: - old_igame.needs_verification = False - self.install_game(old_igame) - - self.log.debug('Removing repair file.') - os.remove(repair_file) - - # check if install tags have changed, if they did; try deleting files that are no longer required. - if old_igame and old_igame.install_tags != igame.install_tags: - old_igame.install_tags = igame.install_tags - self.log.info('Deleting now untagged files.') - self.uninstall_tag(old_igame) - self.install_game(old_igame) - - raise RuntimeError('Nothing to do.') - - res = self.check_installation_conditions(analysis=anlres, install=igame, game=game, - updating=self.is_installed(app_name), - ignore_space_req=ignore_space_req) - - if res.warnings or res.failures: - self.log.info('Installation requirements check returned the following results:') - - return dlm, anlres, game, igame, repair, repair_file, res - - @staticmethod - def check_installation_conditions(analysis: AnalysisResult, - install: InstalledGame, - game: Game, - updating: bool = False, - ignore_space_req: bool = False) -> ConditionCheckResult: - results = ConditionCheckResult(failures=set(), warnings=set()) - - # if on linux, check for eac in the files - if os.name != 'nt': - for f in analysis.manifest_comparison.added: - flower = f.lower() - if 'easyanticheat' in flower: - results.warnings.add('(Linux) This game uses EasyAntiCheat and may not run on linux') - elif 'beclient' in flower: - results.warnings.add('(Linux) This game uses BattlEye and may not run on linux') - elif 'equ8.dll' in flower: - results.warnings.add('(Linux) This game is using EQU8 anticheat and may not run on linux') - elif flower == 'fna.dll' or flower == 'xna.dll': - results.warnings.add('(Linux) This game is using XNA/FNA and may not run in WINE') - - if install.requires_ot: - results.warnings.add('This game requires an ownership verification token and likely uses Denuvo DRM.') - if not install.can_run_offline: - results.warnings.add('This game is not marked for offline use (may still work).') - - # check if enough disk space is free (dl size is the approximate amount the installation will grow) - min_disk_space = analysis.install_size - if updating: - min_disk_space += analysis.biggest_file_size - - # todo when resuming, only check remaining files - _, _, free = shutil.disk_usage(os.path.split(install.install_path)[0]) - if free < min_disk_space: - free_mib = free / 1024 / 1024 - required_mib = min_disk_space / 1024 / 1024 - if ignore_space_req: - results.warnings.add(f'Potentially not enough available disk space! ' - f'{free_mib:.02f} MiB < {required_mib:.02f} MiB') - else: - results.failures.add(f'Not enough available disk space! ' - f'{free_mib:.02f} MiB < {required_mib:.02f} MiB') - - # check if the game actually ships the files or just a uplay installer + packed game files - executables = [f for f in analysis.manifest_comparison.added if - f.lower().endswith('.exe') and not f.startswith('Installer/')] - if not updating and not any('uplay' not in e.lower() for e in executables) and \ - any('uplay' in e.lower() for e in executables): - results.failures.add('This game requires installation via Uplay and does not ship executable game files.') - - # check if the game launches via uplay - if install.executable == 'UplayLaunch.exe': - results.warnings.add('This game requires launching via Uplay, it is recommended to install the game ' - 'via Uplay instead.') - - # check if the game requires linking to an external account first - partner_link = game.metadata.get('customAttributes', {}).get('partnerLinkType', {}).get('value', None) - if partner_link == 'ubisoft': - results.warnings.add('This game requires linking to and activating on a Ubisoft account first, ' - 'this is not currently supported.') - elif partner_link: - results.warnings.add(f'This game requires linking to "{partner_link}", ' - f'this is currently unsupported and the game may not work.') - - return results - - def clean_post_install(self, game: Game, igame: InstalledGame, repair: bool = False, repair_file: str = ''): - old_igame = self.get_installed_game(game.app_name) - if old_igame and repair and os.path.exists(repair_file): - if old_igame.needs_verification: - old_igame.needs_verification = False - self.install_game(old_igame) - - self.log.debug('Removing repair file.') - os.remove(repair_file) - - # check if install tags have changed, if they did; try deleting files that are no longer required. - if old_igame and old_igame.install_tags != igame.install_tags: - old_igame.install_tags = igame.install_tags - self.log.info('Deleting now untagged files.') - self.uninstall_tag(old_igame) - self.install_game(old_igame) - - def get_default_install_dir(self): - return os.path.expanduser(self.lgd.config.get('Legendary', 'install_dir', fallback='~/legendary')) - - def install_game(self, installed_game: InstalledGame) -> dict: - if self.egl_sync_enabled and not installed_game.is_dlc: - if not installed_game.egl_guid: - installed_game.egl_guid = str(uuid4()).replace('-', '').upper() - prereq = self._install_game(installed_game) - self.egl_export(installed_game.app_name) - return prereq - else: - return self._install_game(installed_game) - - def _install_game(self, installed_game: InstalledGame) -> dict: - """Save game metadata and info to mark it "installed" and also show the user the prerequisites""" - self.lgd.set_installed_game(installed_game.app_name, installed_game) - if installed_game.prereq_info: - if not installed_game.prereq_info.get('installed', False): - return installed_game.prereq_info - - return dict() - - def uninstall_game(self, installed_game: InstalledGame, delete_files=True, delete_root_directory=False): - if installed_game.egl_guid: - self.egl_uninstall(installed_game, delete_files=delete_files) - - if delete_files: - try: - manifest = self.load_manifest(self.get_installed_manifest(installed_game.app_name)[0]) - filelist = [fm.filename for fm in manifest.file_manifest_list.elements] - if not delete_filelist(installed_game.install_path, filelist, delete_root_directory): - self.log.error(f'Deleting "{installed_game.install_path}" failed, please remove manually.') - except Exception as e: - self.log.error(f'Deleting failed with {e!r}, please remove {installed_game.install_path} manually.') - - self.lgd.remove_installed_game(installed_game.app_name) - - def uninstall_tag(self, installed_game: InstalledGame): - manifest = self.load_manifest(self.get_installed_manifest(installed_game.app_name)[0]) - tags = installed_game.install_tags - if '' not in tags: - tags.append('') - - # Create list of files that are now no longer needed *and* actually exist on disk - filelist = [ - fm.filename for fm in manifest.file_manifest_list.elements if - not any(((fit in fm.install_tags) or (not fit and not fm.install_tags)) for fit in tags) - and os.path.exists(os.path.join(installed_game.install_path, fm.filename)) - ] - - if not delete_filelist(installed_game.install_path, filelist): - self.log.warning(f'Deleting some deselected files failed, please check/remove manually.') - - def prereq_installed(self, app_name): - igame = self.lgd.get_installed_game(app_name) - igame.prereq_info['installed'] = True - self.lgd.set_installed_game(app_name, igame) - - def import_game(self, game: Game, app_path: str, egl_guid='') -> (Manifest, InstalledGame): - needs_verify = True - manifest_data = None - - # check if the game is from an EGL installation, load manifest if possible - if os.path.exists(os.path.join(app_path, '.egstore')): - mf = None - if not egl_guid: - for f in os.listdir(os.path.join(app_path, '.egstore')): - if not f.endswith('.mancpn'): - continue - - self.log.debug(f'Checking mancpn file "{f}"...') - mancpn = json.load(open(os.path.join(app_path, '.egstore', f), 'rb')) - if mancpn['AppName'] == game.app_name: - self.log.info('Found EGL install metadata, verifying...') - mf = f.replace('.mancpn', '.manifest') - break - else: - mf = f'{egl_guid}.manifest' - - if mf and os.path.exists(os.path.join(app_path, '.egstore', mf)): - manifest_data = open(os.path.join(app_path, '.egstore', mf), 'rb').read() - else: - self.log.warning('.egstore folder exists but manifest file is missing, continuing as regular import...') - - # If there's no in-progress installation assume the game doesn't need to be verified - if mf and not os.path.exists(os.path.join(app_path, '.egstore', 'bps')): - needs_verify = False - if os.path.exists(os.path.join(app_path, '.egstore', 'Pending')): - if os.listdir(os.path.join(app_path, '.egstore', 'Pending')): - needs_verify = True - - if not needs_verify: - self.log.debug(f'No in-progress installation found, assuming complete...') - - if not manifest_data: - self.log.info(f'Downloading latest manifest for "{game.app_name}"') - manifest_data, base_urls = self.get_cdn_manifest(game) - if not game.base_urls: - game.base_urls = base_urls - self.lgd.set_game_meta(game.app_name, game) - else: - # base urls being empty isn't an issue, they'll be fetched when updating/repairing the game - base_urls = game.base_urls - - # parse and save manifest to disk for verification step of import - new_manifest = self.load_manifest(manifest_data) - self.lgd.save_manifest(game.app_name, manifest_data, - version=new_manifest.meta.build_version) - install_size = sum(fm.file_size for fm in new_manifest.file_manifest_list.elements) - - prereq = None - if new_manifest.meta.prereq_ids: - prereq = dict(ids=new_manifest.meta.prereq_ids, name=new_manifest.meta.prereq_name, - path=new_manifest.meta.prereq_path, args=new_manifest.meta.prereq_args) - - offline = game.metadata.get('customAttributes', {}).get('CanRunOffline', {}).get('value', 'true') - ot = game.metadata.get('customAttributes', {}).get('OwnershipToken', {}).get('value', 'false') - igame = InstalledGame(app_name=game.app_name, title=game.app_title, prereq_info=prereq, base_urls=base_urls, - install_path=app_path, version=new_manifest.meta.build_version, is_dlc=game.is_dlc, - executable=new_manifest.meta.launch_exe, can_run_offline=offline == 'true', - launch_parameters=new_manifest.meta.launch_command, requires_ot=ot == 'true', - needs_verification=needs_verify, install_size=install_size, egl_guid=egl_guid) - - return new_manifest, igame - - def egl_get_importable(self): - return [g for g in self.egl.get_manifests() - if not self.is_installed(g.app_name) and - g.main_game_appname == g.app_name and - self.asset_valid(g.app_name)] - - def egl_get_exportable(self): - if not self.egl.manifests: - self.egl.read_manifests() - return [g for g in self.get_installed_list() if g.app_name not in self.egl.manifests] - - def egl_import(self, app_name): - if not self.asset_valid(app_name): - raise ValueError(f'To-be-imported game {app_name} not in game asset database!') - - self.log.debug(f'Importing "{app_name}" from EGL') - # load egl json file - try: - egl_game = self.egl.get_manifest(app_name=app_name) - except ValueError: - self.log.fatal(f'EGL Manifest for {app_name} could not be loaded, not importing!') - return - # convert egl json file - lgd_igame = egl_game.to_lgd_igame() - - # fix path on Linux if the game is installed to a Windows drive mapping - if os.name != 'nt' and not lgd_igame.install_path.startswith('/'): - drive_letter = lgd_igame.install_path[:2].lower() - drive_c_path = self.egl.programdata_path.partition('ProgramData')[0] - wine_pfx = os.path.realpath(os.path.join(drive_c_path, '..')) - mapped_path = os.path.realpath(os.path.join(wine_pfx, 'dosdevices', drive_letter)) - if 'dosdevices' in mapped_path: - self.log.error(f'Unable to resolve path for mapped drive "{drive_letter}" ' - f'for WINE prefix at "{wine_pfx}"') - return - - game_path = lgd_igame.install_path[2:].replace('\\', '/').lstrip('/') - new_path = os.path.realpath(os.path.join(mapped_path, game_path)) - self.log.info(f'Adjusted game install path from "{lgd_igame.install_path}" to "{new_path}"') - lgd_igame.install_path = new_path - - # check if manifest exists - manifest_filename = os.path.join(lgd_igame.install_path, '.egstore', f'{lgd_igame.egl_guid}.manifest') - if not os.path.exists(manifest_filename): - self.log.warning(f'Game Manifest "{manifest_filename}" not found, cannot import!') - return - - # load manifest file and copy it over - with open(manifest_filename, 'rb') as f: - manifest_data = f.read() - new_manifest = self.load_manifest(manifest_data) - self.lgd.save_manifest(lgd_igame.app_name, manifest_data, - version=new_manifest.meta.build_version) - - # transfer install tag choices to config - if lgd_igame.install_tags: - if app_name not in self.lgd.config: - self.lgd.config[app_name] = dict() - self.lgd.config.set(app_name, 'install_tags', ','.join(lgd_igame.install_tags)) - - # mark game as installed - _ = self._install_game(lgd_igame) - return - - def egl_export(self, app_name): - self.log.debug(f'Exporting "{app_name}" to EGL') - # load igame/game - lgd_game = self.get_game(app_name) - lgd_igame = self._get_installed_game(app_name) - manifest_data, _ = self.get_installed_manifest(app_name) - if not manifest_data: - self.log.error(f'Game Manifest for "{app_name}" not found, cannot export!') - return - - # create guid if it's not set already - if not lgd_igame.egl_guid: - lgd_igame.egl_guid = str(uuid4()).replace('-', '').upper() - _ = self._install_game(lgd_igame) - # convert to egl manifest - egl_game = EGLManifest.from_lgd_game(lgd_game, lgd_igame) - - # make sure .egstore folder exists - egstore_folder = os.path.join(lgd_igame.install_path, '.egstore') - if not os.path.exists(egstore_folder): - os.makedirs(egstore_folder) - - # copy manifest and create mancpn file in .egstore folder - with open(os.path.join(egstore_folder, f'{egl_game.installation_guid}.manifest', ), 'wb') as mf: - mf.write(manifest_data) - - mancpn = dict(FormatVersion=0, AppName=app_name, - CatalogItemId=lgd_game.asset_info.catalog_item_id, - CatalogNamespace=lgd_game.asset_info.namespace) - with open(os.path.join(egstore_folder, f'{egl_game.installation_guid}.mancpn', ), 'w') as mcpnf: - json.dump(mancpn, mcpnf, indent=4, sort_keys=True) - - # And finally, write the file for EGL - self.egl.set_manifest(egl_game) - - def egl_uninstall(self, igame: InstalledGame, delete_files=True): - try: - self.egl.delete_manifest(igame.app_name) - except ValueError as e: - self.log.warning(f'Deleting EGL manifest failed: {e!r}') - - if delete_files: - delete_folder(os.path.join(igame.install_path, '.egstore')) - - def egl_restore_or_uninstall(self, igame): - # check if game binary is still present, if not; uninstall - if not os.path.exists(os.path.join(igame.install_path, - igame.executable.lstrip('/'))): - self.log.warning('Synced game\'s files no longer exists, assuming it has been uninstalled.') - igame.egl_guid = '' - return self.uninstall_game(igame, delete_files=False) - else: - self.log.info('Game files exist, assuming game is still installed, re-exporting to EGL...') - return self.egl_export(igame.app_name) - - def egl_sync(self, app_name=''): - """ - Sync game installs between Legendary and the Epic Games Launcher - """ - # read egl json files - if app_name: - lgd_igame = self._get_installed_game(app_name) - if not self.egl.manifests: - self.egl.read_manifests() - - if app_name not in self.egl.manifests: - self.log.info(f'Synced app "{app_name}" is no longer in the EGL manifest list.') - return self.egl_restore_or_uninstall(lgd_igame) - else: - egl_igame = self.egl.get_manifest(app_name) - if (egl_igame.app_version_string != lgd_igame.version) or \ - (egl_igame.install_tags != lgd_igame.install_tags): - self.log.info(f'App "{egl_igame.app_name}" has been updated from EGL, syncing...') - return self.egl_import(egl_igame.app_name) - else: - # check EGL -> Legendary sync - for egl_igame in self.egl.get_manifests(): - if egl_igame.main_game_appname != egl_igame.app_name: # skip DLC - continue - if not self.asset_valid(egl_igame.app_name): # skip non-owned games - continue - - if not self._is_installed(egl_igame.app_name): - self.egl_import(egl_igame.app_name) - else: - lgd_igame = self._get_installed_game(egl_igame.app_name) - if (egl_igame.app_version_string != lgd_igame.version) or \ - (egl_igame.install_tags != lgd_igame.install_tags): - self.log.info(f'App "{egl_igame.app_name}" has been updated from EGL, syncing...') - self.egl_import(egl_igame.app_name) - - # Check for games that have been uninstalled - for lgd_igame in self._get_installed_list(): - if not lgd_igame.egl_guid: # skip non-exported - continue - if lgd_igame.app_name in self.egl.manifests: - continue - - self.log.info(f'Synced app "{lgd_igame.app_name}" is no longer in the EGL manifest list.') - self.egl_restore_or_uninstall(lgd_igame) - - @property - def egl_sync_enabled(self): - return self.lgd.config.getboolean('Legendary', 'egl_sync', fallback=False) - - def exit(self): - """ - Do cleanup, config saving, and exit. - """ - self.lgd.save_config() diff --git a/custom_legendary/downloader/__init__.py b/custom_legendary/downloader/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/custom_legendary/downloader/manager.py b/custom_legendary/downloader/manager.py deleted file mode 100644 index 63e84128..00000000 --- a/custom_legendary/downloader/manager.py +++ /dev/null @@ -1,763 +0,0 @@ -# coding: utf-8 - -# please don't look at this code too hard, it's a mess. - -import logging -import os -import time -from collections import Counter, defaultdict, deque -from logging.handlers import QueueHandler -from multiprocessing import cpu_count, Process, Queue as MPQueue -from multiprocessing.shared_memory import SharedMemory -from queue import Empty -from sys import exit -from threading import Condition, Thread - -from custom_legendary.downloader.workers import DLWorker, FileWorker -from custom_legendary.models.downloading import * -from custom_legendary.models.manifest import ManifestComparison, Manifest - - -class DLManager(Process): - def __init__(self, download_dir, base_url, cache_dir=None, status_q=None, - max_workers=0, update_interval=1.0, dl_timeout=10, resume_file=None, - max_shared_memory=1024 * 1024 * 1024): - super().__init__(name='DLManager') - self.log = logging.getLogger('DLM') - self.proc_debug = False - - self.base_url = base_url - self.dl_dir = download_dir - self.cache_dir = cache_dir if cache_dir else os.path.join(download_dir, '.cache') - - # All the queues! - self.logging_queue = None - self.dl_worker_queue = None - self.writer_queue = None - self.dl_result_q = None - self.writer_result_q = None - self.max_workers = max_workers if max_workers else min(cpu_count() * 2, 16) - self.dl_timeout = dl_timeout - - # Analysis stuff - self.analysis = None - self.tasks = deque() - self.chunks_to_dl = deque() - self.chunk_data_list = None - - # shared memory stuff - self.max_shared_memory = max_shared_memory # 1 GiB by default - self.sms = deque() - self.shared_memory = None - - # Interval for log updates and pushing updates to the queue - self.update_interval = update_interval - self.status_queue = status_q # queue used to relay status info back to GUI/CLI - - # Resume file stuff - self.resume_file = resume_file - self.hash_map = dict() - - # cross-thread runtime information - self.running = True - self.active_tasks = 0 - self.children = [] - self.threads = [] - self.conditions = [] - # bytes downloaded and decompressed since last report - self.bytes_downloaded_since_last = 0 - self.bytes_decompressed_since_last = 0 - # bytes written since last report - self.bytes_written_since_last = 0 - # bytes read since last report - self.bytes_read_since_last = 0 - # chunks written since last report - self.num_processed_since_last = 0 - self.num_tasks_processed_since_last = 0 - - def run_analysis(self, manifest: Manifest, old_manifest: Manifest = None, - patch=True, resume=True, file_prefix_filter=None, - file_exclude_filter=None, file_install_tag=None, - processing_optimization=False) -> AnalysisResult: - """ - Run analysis on manifest and old manifest (if not None) and return a result - with a summary resources required in order to install the provided manifest. - - :param manifest: Manifest to install - :param old_manifest: Old manifest to patch from (if applicable) - :param patch: Patch instead of redownloading the entire file - :param resume: Continue based on resume file if it exists - :param file_prefix_filter: Only download files that start with this prefix - :param file_exclude_filter: Exclude files with this prefix from download - :param file_install_tag: Only install files with the specified tag - :param processing_optimization: Attempt to optimize processing order and RAM usage - :return: AnalysisResult - """ - - analysis_res = AnalysisResult() - analysis_res.install_size = sum(fm.file_size for fm in manifest.file_manifest_list.elements) - analysis_res.biggest_chunk = max(c.window_size for c in manifest.chunk_data_list.elements) - analysis_res.biggest_file_size = max(f.file_size for f in manifest.file_manifest_list.elements) - is_1mib = analysis_res.biggest_chunk == 1024 * 1024 - self.log.debug(f'Biggest chunk size: {analysis_res.biggest_chunk} bytes (== 1 MiB? {is_1mib})') - - self.log.debug(f'Creating manifest comparison...') - mc = ManifestComparison.create(manifest, old_manifest) - analysis_res.manifest_comparison = mc - - if resume and self.resume_file and os.path.exists(self.resume_file): - self.log.info('Found previously interrupted download. Download will be resumed if possible.') - try: - missing = 0 - mismatch = 0 - completed_files = set() - - for line in open(self.resume_file).readlines(): - file_hash, _, filename = line.strip().partition(':') - _p = os.path.join(self.dl_dir, filename) - if not os.path.exists(_p): - self.log.debug(f'File does not exist but is in resume file: "{_p}"') - missing += 1 - elif file_hash != manifest.file_manifest_list.get_file_by_path(filename).sha_hash.hex(): - mismatch += 1 - else: - completed_files.add(filename) - - if missing: - self.log.warning(f'{missing} previously completed file(s) are missing, they will be redownloaded.') - if mismatch: - self.log.warning(f'{mismatch} existing file(s) have been changed and will be redownloaded.') - - # remove completed files from changed/added and move them to unchanged for the analysis. - mc.added -= completed_files - mc.changed -= completed_files - mc.unchanged |= completed_files - self.log.info(f'Skipping {len(completed_files)} files based on resume data.') - except Exception as e: - self.log.warning(f'Reading resume file failed: {e!r}, continuing as normal...') - - # Install tags are used for selective downloading, e.g. for language packs - additional_deletion_tasks = [] - if file_install_tag is not None: - if isinstance(file_install_tag, str): - file_install_tag = [file_install_tag] - - files_to_skip = set(i.filename for i in manifest.file_manifest_list.elements - if not any((fit in i.install_tags) or (not fit and not i.install_tags) - for fit in file_install_tag)) - self.log.info(f'Found {len(files_to_skip)} files to skip based on install tag.') - mc.added -= files_to_skip - mc.changed -= files_to_skip - mc.unchanged |= files_to_skip - for fname in sorted(files_to_skip): - additional_deletion_tasks.append(FileTask(fname, delete=True, silent=True)) - - # if include/exclude prefix has been set: mark all files that are not to be downloaded as unchanged - if file_exclude_filter: - if isinstance(file_exclude_filter, str): - file_exclude_filter = [file_exclude_filter] - - file_exclude_filter = [f.lower() for f in file_exclude_filter] - files_to_skip = set(i.filename for i in manifest.file_manifest_list.elements if - any(i.filename.lower().startswith(pfx) for pfx in file_exclude_filter)) - self.log.info(f'Found {len(files_to_skip)} files to skip based on exclude prefix.') - mc.added -= files_to_skip - mc.changed -= files_to_skip - mc.unchanged |= files_to_skip - - if file_prefix_filter: - if isinstance(file_prefix_filter, str): - file_prefix_filter = [file_prefix_filter] - - file_prefix_filter = [f.lower() for f in file_prefix_filter] - files_to_skip = set(i.filename for i in manifest.file_manifest_list.elements if not - any(i.filename.lower().startswith(pfx) for pfx in file_prefix_filter)) - self.log.info(f'Found {len(files_to_skip)} files to skip based on include prefix(es)') - mc.added -= files_to_skip - mc.changed -= files_to_skip - mc.unchanged |= files_to_skip - - if file_prefix_filter or file_exclude_filter or file_install_tag: - self.log.info(f'Remaining files after filtering: {len(mc.added) + len(mc.changed)}') - # correct install size after filtering - analysis_res.install_size = sum(fm.file_size for fm in manifest.file_manifest_list.elements - if fm.filename in mc.added) - - if mc.removed: - analysis_res.removed = len(mc.removed) - self.log.debug(f'{analysis_res.removed} removed files') - if mc.added: - analysis_res.added = len(mc.added) - self.log.debug(f'{analysis_res.added} added files') - if mc.changed: - analysis_res.changed = len(mc.changed) - self.log.debug(f'{analysis_res.changed} changed files') - if mc.unchanged: - analysis_res.unchanged = len(mc.unchanged) - self.log.debug(f'{analysis_res.unchanged} unchanged files') - - if processing_optimization and len(manifest.file_manifest_list.elements) > 100_000: - self.log.warning('Manifest contains too many files, processing optimizations will be disabled.') - processing_optimization = False - elif processing_optimization: - self.log.info('Processing order optimization is enabled, analysis may take a few seconds longer...') - - # count references to chunks for determining runtime cache size later - references = Counter() - fmlist = sorted(manifest.file_manifest_list.elements, - key=lambda a: a.filename.lower()) - - for fm in fmlist: - self.hash_map[fm.filename] = fm.sha_hash.hex() - - # chunks of unchanged files are not downloaded so we can skip them - if fm.filename in mc.unchanged: - analysis_res.unchanged += fm.file_size - continue - - for cp in fm.chunk_parts: - references[cp.guid_num] += 1 - - if processing_optimization: - s_time = time.time() - # reorder the file manifest list to group files that share many chunks - # 4 is mostly arbitrary but has shown in testing to be a good choice - min_overlap = 4 - # ignore files with less than N chunk parts, this speeds things up dramatically - cp_threshold = 5 - - remaining_files = {fm.filename: {cp.guid_num for cp in fm.chunk_parts} - for fm in fmlist if fm.filename not in mc.unchanged} - _fmlist = [] - - # iterate over all files that will be downloaded and pair up those that share the most chunks - for fm in fmlist: - if fm.filename not in remaining_files: - continue - - _fmlist.append(fm) - f_chunks = remaining_files.pop(fm.filename) - if len(f_chunks) < cp_threshold: - continue - - best_overlap, match = 0, None - for fname, chunks in remaining_files.items(): - if len(chunks) < cp_threshold: - continue - overlap = len(f_chunks & chunks) - if overlap > min_overlap and overlap > best_overlap: - best_overlap, match = overlap, fname - - if match: - _fmlist.append(manifest.file_manifest_list.get_file_by_path(match)) - remaining_files.pop(match) - - fmlist = _fmlist - opt_delta = time.time() - s_time - self.log.debug(f'Processing optimizations took {opt_delta:.01f} seconds.') - - # determine reusable chunks and prepare lookup table for reusable ones - re_usable = defaultdict(dict) - if old_manifest and mc.changed and patch: - self.log.debug('Analyzing manifests for re-usable chunks...') - for changed in mc.changed: - old_file = old_manifest.file_manifest_list.get_file_by_path(changed) - new_file = manifest.file_manifest_list.get_file_by_path(changed) - - existing_chunks = defaultdict(list) - off = 0 - for cp in old_file.chunk_parts: - existing_chunks[cp.guid_num].append((off, cp.offset, cp.offset + cp.size)) - off += cp.size - - for cp in new_file.chunk_parts: - key = (cp.guid_num, cp.offset, cp.size) - for file_o, cp_o, cp_end_o in existing_chunks[cp.guid_num]: - # check if new chunk part is wholly contained in the old chunk part - if cp_o <= cp.offset and (cp.offset + cp.size) <= cp_end_o: - references[cp.guid_num] -= 1 - re_usable[changed][key] = file_o + (cp.offset - cp_o) - analysis_res.reuse_size += cp.size - break - - last_cache_size = current_cache_size = 0 - # set to determine whether a file is currently cached or not - cached = set() - # Using this secondary set is orders of magnitude faster than checking the deque. - chunks_in_dl_list = set() - # This is just used to count all unique guids that have been cached - dl_cache_guids = set() - - # run through the list of files and create the download jobs and also determine minimum - # runtime cache requirement by simulating adding/removing from cache during download. - self.log.debug('Creating filetasks and chunktasks...') - for current_file in fmlist: - # skip unchanged and empty files - if current_file.filename in mc.unchanged: - continue - elif not current_file.chunk_parts: - self.tasks.append(FileTask(current_file.filename, empty=True)) - continue - - existing_chunks = re_usable.get(current_file.filename, None) - chunk_tasks = [] - reused = 0 - - for cp in current_file.chunk_parts: - ct = ChunkTask(cp.guid_num, cp.offset, cp.size) - - # re-use the chunk from the existing file if we can - if existing_chunks and (cp.guid_num, cp.offset, cp.size) in existing_chunks: - reused += 1 - ct.chunk_file = current_file.filename - ct.chunk_offset = existing_chunks[(cp.guid_num, cp.offset, cp.size)] - else: - # add to DL list if not already in it - if cp.guid_num not in chunks_in_dl_list: - self.chunks_to_dl.append(cp.guid_num) - chunks_in_dl_list.add(cp.guid_num) - - # if chunk has more than one use or is already in cache, - # check if we need to add or remove it again. - if references[cp.guid_num] > 1 or cp.guid_num in cached: - references[cp.guid_num] -= 1 - - # delete from cache if no references left - if references[cp.guid_num] < 1: - current_cache_size -= analysis_res.biggest_chunk - cached.remove(cp.guid_num) - ct.cleanup = True - # add to cache if not already cached - elif cp.guid_num not in cached: - dl_cache_guids.add(cp.guid_num) - cached.add(cp.guid_num) - current_cache_size += analysis_res.biggest_chunk - else: - ct.cleanup = True - - chunk_tasks.append(ct) - - if reused: - self.log.debug(f' + Reusing {reused} chunks from: {current_file.filename}') - # open temporary file that will contain download + old file contents - self.tasks.append(FileTask(current_file.filename + u'.tmp', fopen=True)) - self.tasks.extend(chunk_tasks) - self.tasks.append(FileTask(current_file.filename + u'.tmp', close=True)) - # delete old file and rename temporary - self.tasks.append(FileTask(current_file.filename, delete=True, rename=True, - temporary_filename=current_file.filename + u'.tmp')) - else: - self.tasks.append(FileTask(current_file.filename, fopen=True)) - self.tasks.extend(chunk_tasks) - self.tasks.append(FileTask(current_file.filename, close=True)) - - # check if runtime cache size has changed - if current_cache_size > last_cache_size: - self.log.debug(f' * New maximum cache size: {current_cache_size / 1024 / 1024:.02f} MiB') - last_cache_size = current_cache_size - - self.log.debug(f'Final cache size requirement: {last_cache_size / 1024 / 1024} MiB.') - analysis_res.min_memory = last_cache_size + (1024 * 1024 * 32) # add some padding just to be safe - - # Todo implement on-disk caching to avoid this issue. - if analysis_res.min_memory > self.max_shared_memory: - shared_mib = f'{self.max_shared_memory / 1024 / 1024:.01f} MiB' - required_mib = f'{analysis_res.min_memory / 1024 / 1024:.01f} MiB' - suggested_mib = round(self.max_shared_memory / 1024 / 1024 + - (analysis_res.min_memory - self.max_shared_memory) / 1024 / 1024 + 32) - - if processing_optimization: - message = f'Try running legendary with "--enable-reordering --max-shared-memory {suggested_mib:.0f}"' - else: - message = 'Try running legendary with "--enable-reordering" to reduce memory usage, ' \ - f'or use "--max-shared-memory {suggested_mib:.0f}" to increase the limit.' - - raise MemoryError(f'Current shared memory cache is smaller than required: {shared_mib} < {required_mib}. ' - + message) - - # calculate actual dl and patch write size. - analysis_res.dl_size = \ - sum(c.file_size for c in manifest.chunk_data_list.elements if c.guid_num in chunks_in_dl_list) - analysis_res.uncompressed_dl_size = \ - sum(c.window_size for c in manifest.chunk_data_list.elements if c.guid_num in chunks_in_dl_list) - - # add jobs to remove files - for fname in mc.removed: - self.tasks.append(FileTask(fname, delete=True)) - self.tasks.extend(additional_deletion_tasks) - - analysis_res.num_chunks_cache = len(dl_cache_guids) - self.chunk_data_list = manifest.chunk_data_list - self.analysis = analysis_res - - return analysis_res - - def download_job_manager(self, task_cond: Condition, shm_cond: Condition): - while self.chunks_to_dl and self.running: - while self.active_tasks < self.max_workers * 2 and self.chunks_to_dl: - try: - sms = self.sms.popleft() - no_shm = False - except IndexError: # no free cache - no_shm = True - break - - c_guid = self.chunks_to_dl.popleft() - chunk = self.chunk_data_list.get_chunk_by_guid(c_guid) - self.log.debug(f'Adding {chunk.guid_num} (active: {self.active_tasks})') - try: - self.dl_worker_queue.put(DownloaderTask(url=self.base_url + '/' + chunk.path, - chunk_guid=c_guid, shm=sms), - timeout=1.0) - except Exception as e: - self.log.warning(f'Failed to add to download queue: {e!r}') - self.chunks_to_dl.appendleft(c_guid) - break - - self.active_tasks += 1 - else: - # active tasks limit hit, wait for tasks to finish - with task_cond: - self.log.debug('Waiting for download tasks to complete..') - task_cond.wait(timeout=1.0) - continue - - if no_shm: - # if we break we ran out of shared memory, so wait for that. - with shm_cond: - self.log.debug('Waiting for more shared memory...') - shm_cond.wait(timeout=1.0) - - self.log.debug('Download Job Manager quitting...') - - def dl_results_handler(self, task_cond: Condition): - in_buffer = dict() - - task = self.tasks.popleft() - current_file = '' - - while task and self.running: - if isinstance(task, FileTask): # this wasn't necessarily a good idea... - try: - if task.empty: - self.writer_queue.put(WriterTask(task.filename, empty=True), timeout=1.0) - elif task.rename: - self.writer_queue.put(WriterTask(task.filename, rename=True, - delete=task.delete, - old_filename=task.temporary_filename), - timeout=1.0) - elif task.delete: - self.writer_queue.put(WriterTask(task.filename, delete=True, silent=task.silent), timeout=1.0) - elif task.open: - self.writer_queue.put(WriterTask(task.filename, fopen=True), timeout=1.0) - current_file = task.filename - elif task.close: - self.writer_queue.put(WriterTask(task.filename, close=True), timeout=1.0) - except Exception as e: - self.tasks.appendleft(task) - self.log.warning(f'Adding to queue failed: {e!r}') - continue - - try: - task = self.tasks.popleft() - except IndexError: # finished - break - continue - - while (task.chunk_guid in in_buffer) or task.chunk_file: - res_shm = None - if not task.chunk_file: # not re-using from an old file - res_shm = in_buffer[task.chunk_guid].shm - - try: - self.log.debug(f'Adding {task.chunk_guid} to writer queue') - self.writer_queue.put(WriterTask( - filename=current_file, shared_memory=res_shm, - chunk_offset=task.chunk_offset, chunk_size=task.chunk_size, - chunk_guid=task.chunk_guid, release_memory=task.cleanup, - old_file=task.chunk_file # todo on-disk cache - ), timeout=1.0) - except Exception as e: - self.log.warning(f'Adding to queue failed: {e!r}') - break - - if task.cleanup and not task.chunk_file: - del in_buffer[task.chunk_guid] - - try: - task = self.tasks.popleft() - if isinstance(task, FileTask): - break - except IndexError: # finished - task = None - break - else: # only enter blocking code if the loop did not break - try: - res = self.dl_result_q.get(timeout=1) - self.active_tasks -= 1 - with task_cond: - task_cond.notify() - - if res.success: - self.log.debug(f'Download for {res.guid} succeeded, adding to in_buffer...') - in_buffer[res.guid] = res - self.bytes_downloaded_since_last += res.compressed_size - self.bytes_decompressed_since_last += res.size - else: - self.log.error(f'Download for {res.guid} failed, retrying...') - try: - self.dl_worker_queue.put(DownloaderTask( - url=res.url, chunk_guid=res.guid, shm=res.shm - ), timeout=1.0) - self.active_tasks += 1 - except Exception as e: - self.log.warning(f'Failed adding retry task to queue! {e!r}') - # If this failed for whatever reason, put the chunk at the front of the DL list - self.chunks_to_dl.appendleft(res.chunk_guid) - except Empty: - pass - except Exception as e: - self.log.warning(f'Unhandled exception when trying to read download result queue: {e!r}') - - self.log.debug('Download result handler quitting...') - - def fw_results_handler(self, shm_cond: Condition): - while self.running: - try: - res = self.writer_result_q.get(timeout=1.0) - self.num_tasks_processed_since_last += 1 - - if res.closed and self.resume_file and res.success: - if res.filename.endswith('.tmp'): - res.filename = res.filename[:-4] - - file_hash = self.hash_map[res.filename] - # write last completed file to super simple resume file - with open(self.resume_file, 'ab') as rf: - rf.write(f'{file_hash}:{res.filename}\n'.encode('utf-8')) - - if res.kill: - self.log.debug('Got termination command in FW result handler') - break - - if not res.success: - # todo make this kill the installation process or at least skip the file and mark it as failed - self.log.fatal(f'Writing for {res.filename} failed!') - if res.release_memory: - self.sms.appendleft(res.shm) - with shm_cond: - shm_cond.notify() - - if res.chunk_guid: - self.bytes_written_since_last += res.size - # if there's no shared memory we must have read from disk. - if not res.shm: - self.bytes_read_since_last += res.size - self.num_processed_since_last += 1 - - except Empty: - continue - except Exception as e: - self.log.warning(f'Exception when trying to read writer result queue: {e!r}') - self.log.debug('Writer result handler quitting...') - - def run(self): - if not self.analysis: - raise ValueError('Did not run analysis before trying to run download!') - - # Subprocess will use its own root logger that logs to a Queue instead - _root = logging.getLogger() - _root.setLevel(logging.DEBUG if self.proc_debug else logging.INFO) - if self.logging_queue: - _root.handlers = [] - _root.addHandler(QueueHandler(self.logging_queue)) - - self.log = logging.getLogger('DLManager') - self.log.info(f'Download Manager running with process-id: {os.getpid()}') - - try: - self.run_real() - except KeyboardInterrupt: - self.log.warning('Immediate exit requested!') - self.running = False - - # send conditions to unlock threads if they aren't already - for cond in self.conditions: - with cond: - cond.notify() - - # make sure threads are dead. - for t in self.threads: - t.join(timeout=5.0) - if t.is_alive(): - self.log.warning(f'Thread did not terminate! {repr(t)}') - - # clean up all the queues, otherwise this process won't terminate properly - for name, q in zip(('Download jobs', 'Writer jobs', 'Download results', 'Writer results'), - (self.dl_worker_queue, self.writer_queue, self.dl_result_q, self.writer_result_q)): - self.log.debug(f'Cleaning up queue "{name}"') - try: - while True: - _ = q.get_nowait() - except Empty: - q.close() - q.join_thread() - - def run_real(self): - self.shared_memory = SharedMemory(create=True, size=self.max_shared_memory) - self.log.debug(f'Created shared memory of size: {self.shared_memory.size / 1024 / 1024:.02f} MiB') - - # create the shared memory segments and add them to their respective pools - for i in range(int(self.shared_memory.size / self.analysis.biggest_chunk)): - _sms = SharedMemorySegment(offset=i * self.analysis.biggest_chunk, - end=i * self.analysis.biggest_chunk + self.analysis.biggest_chunk) - self.sms.append(_sms) - - self.log.debug(f'Created {len(self.sms)} shared memory segments.') - - # Create queues - self.dl_worker_queue = MPQueue(-1) - self.writer_queue = MPQueue(-1) - self.dl_result_q = MPQueue(-1) - self.writer_result_q = MPQueue(-1) - - self.log.info(f'Starting download workers...') - for i in range(self.max_workers): - w = DLWorker(f'DLWorker {i + 1}', self.dl_worker_queue, self.dl_result_q, - self.shared_memory.name, logging_queue=self.logging_queue, - dl_timeout=self.dl_timeout) - self.children.append(w) - w.start() - - self.log.info('Starting file writing worker...') - writer_p = FileWorker(self.writer_queue, self.writer_result_q, self.dl_dir, - self.shared_memory.name, self.cache_dir, self.logging_queue) - self.children.append(writer_p) - writer_p.start() - - num_chunk_tasks = sum(isinstance(t, ChunkTask) for t in self.tasks) - num_dl_tasks = len(self.chunks_to_dl) - num_tasks = len(self.tasks) - num_shared_memory_segments = len(self.sms) - self.log.debug(f'Chunks to download: {num_dl_tasks}, File tasks: {num_tasks}, Chunk tasks: {num_chunk_tasks}') - - # active downloader tasks - self.active_tasks = 0 - processed_chunks = 0 - processed_tasks = 0 - total_dl = 0 - total_write = 0 - - # synchronization conditions - shm_cond = Condition() - task_cond = Condition() - self.conditions = [shm_cond, task_cond] - - # start threads - s_time = time.time() - self.threads.append(Thread(target=self.download_job_manager, args=(task_cond, shm_cond))) - self.threads.append(Thread(target=self.dl_results_handler, args=(task_cond,))) - self.threads.append(Thread(target=self.fw_results_handler, args=(shm_cond,))) - - for t in self.threads: - t.start() - - last_update = time.time() - - while processed_tasks < num_tasks: - delta = time.time() - last_update - if not delta: - time.sleep(self.update_interval) - continue - - # update all the things - processed_chunks += self.num_processed_since_last - processed_tasks += self.num_tasks_processed_since_last - - total_dl += self.bytes_downloaded_since_last - total_write += self.bytes_written_since_last - - dl_speed = self.bytes_downloaded_since_last / delta - dl_unc_speed = self.bytes_decompressed_since_last / delta - w_speed = self.bytes_written_since_last / delta - r_speed = self.bytes_read_since_last / delta - # c_speed = self.num_processed_since_last / delta - - # set temporary counters to 0 - self.bytes_read_since_last = self.bytes_written_since_last = 0 - self.bytes_downloaded_since_last = self.num_processed_since_last = 0 - self.bytes_decompressed_since_last = self.num_tasks_processed_since_last = 0 - last_update = time.time() - - perc = (processed_chunks / num_chunk_tasks) * 100 - runtime = time.time() - s_time - total_avail = len(self.sms) - total_used = (num_shared_memory_segments - total_avail) * (self.analysis.biggest_chunk / 1024 / 1024) - - try: - average_speed = processed_chunks / runtime - estimate = (num_chunk_tasks - processed_chunks) / average_speed - except ZeroDivisionError: - average_speed = estimate = 0 - - # TODO set current_filename argument of UIUpdate - # send status update to back to instantiator (if queue exists) - if self.status_queue: - try: - self.status_queue.put(UIUpdate( - progress=perc, - runtime=round(runtime), - estimated_time_left=round(estimate), - processed_chunks=processed_chunks, - chunk_tasks=num_chunk_tasks, - total_downloaded=total_dl, - total_written=total_write, - cache_usage=total_used, - active_tasks=self.active_tasks, - download_speed=dl_speed, - download_decompressed_speed=dl_unc_speed, - write_speed=w_speed, - read_speed=r_speed, - ), timeout=1.0) - except Exception as e: - self.log.warning(f'Failed to send status update to queue: {e!r}') - - time.sleep(self.update_interval) - - for i in range(self.max_workers): - self.dl_worker_queue.put_nowait(DownloaderTask(kill=True)) - - self.log.info('Waiting for installation to finish...') - self.writer_queue.put_nowait(WriterTask('', kill=True)) - - writer_p.join(timeout=10.0) - if writer_p.exitcode is None: - self.log.warning(f'Terminating writer process, no exit code!') - writer_p.terminate() - - # forcibly kill DL workers that are not actually dead yet - for child in self.children: - if child.exitcode is None: - child.terminate() - - # make sure all the threads are dead. - for t in self.threads: - t.join(timeout=5.0) - if t.is_alive(): - self.log.warning(f'Thread did not terminate! {repr(t)}') - - # clean up resume file - if self.resume_file: - try: - os.remove(self.resume_file) - except OSError as e: - self.log.warning(f'Failed to remove resume file: {e!r}') - - # close up shared memory - self.shared_memory.close() - self.shared_memory.unlink() - self.shared_memory = None - - self.log.info('All done! Download manager quitting...') - # finally, exit the process. - exit(0) diff --git a/custom_legendary/downloader/workers.py b/custom_legendary/downloader/workers.py deleted file mode 100644 index 9944cb70..00000000 --- a/custom_legendary/downloader/workers.py +++ /dev/null @@ -1,276 +0,0 @@ -# coding: utf-8 - -import logging -import os -import time -from logging.handlers import QueueHandler -from multiprocessing import Process -from multiprocessing.shared_memory import SharedMemory -from queue import Empty - -import requests - -from custom_legendary.models.chunk import Chunk -from custom_legendary.models.downloading import DownloaderTaskResult, WriterTaskResult - - -class DLWorker(Process): - def __init__(self, name, queue, out_queue, shm, max_retries=5, - logging_queue=None, dl_timeout=10): - super().__init__(name=name) - self.q = queue - self.o_q = out_queue - self.session = requests.session() - self.session.headers.update({ - 'User-Agent': 'EpicGamesLauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' - }) - self.max_retries = max_retries - self.shm = SharedMemory(name=shm) - self.log_level = logging.getLogger().level - self.logging_queue = logging_queue - self.dl_timeout = float(dl_timeout) if dl_timeout else 10.0 - - def run(self): - # we have to fix up the logger before we can start - _root = logging.getLogger() - _root.handlers = [] - _root.addHandler(QueueHandler(self.logging_queue)) - - logger = logging.getLogger(self.name) - logger.setLevel(self.log_level) - logger.debug(f'Download worker reporting for duty!') - - empty = False - while True: - try: - job = self.q.get(timeout=10.0) - empty = False - except Empty: - if not empty: - logger.debug(f'Queue Empty, waiting for more...') - empty = True - continue - - if job.kill: # let worker die - logger.debug(f'Worker received kill signal, shutting down...') - break - - tries = 0 - dl_start = dl_end = 0 - compressed = 0 - chunk = None - - try: - while tries < self.max_retries: - # print('Downloading', job.url) - logger.debug(f'Downloading {job.url}') - dl_start = time.time() - - try: - r = self.session.get(job.url, timeout=self.dl_timeout) - r.raise_for_status() - except Exception as e: - logger.warning(f'Chunk download for {job.guid} failed: ({e!r}), retrying...') - continue - - dl_end = time.time() - if r.status_code != 200: - logger.warning(f'Chunk download for {job.guid} failed: status {r.status_code}, retrying...') - continue - else: - compressed = len(r.content) - chunk = Chunk.read_buffer(r.content) - break - else: - raise TimeoutError('Max retries reached') - except Exception as e: - logger.error(f'Job for {job.guid} failed with: {e!r}, fetching next one...') - # add failed job to result queue to be requeued - self.o_q.put(DownloaderTaskResult(success=False, chunk_guid=job.guid, shm=job.shm, url=job.url)) - except KeyboardInterrupt: - logger.warning('Immediate exit requested, quitting...') - break - - if not chunk: - logger.warning(f'Chunk somehow None?') - self.o_q.put(DownloaderTaskResult(success=False, chunk_guid=job.guid, shm=job.shm, url=job.url)) - continue - - # decompress stuff - try: - size = len(chunk.data) - if size > job.shm.size: - logger.fatal(f'Downloaded chunk is longer than SharedMemorySegment!') - - self.shm.buf[job.shm.offset:job.shm.offset + size] = bytes(chunk.data) - del chunk - self.o_q.put(DownloaderTaskResult(success=True, chunk_guid=job.guid, shm=job.shm, - url=job.url, size=size, compressed_size=compressed, - time_delta=dl_end - dl_start)) - except Exception as e: - logger.warning(f'Job for {job.guid} failed with: {e!r}, fetching next one...') - self.o_q.put(DownloaderTaskResult(success=False, chunk_guid=job.guid, shm=job.shm, url=job.url)) - continue - except KeyboardInterrupt: - logger.warning('Immediate exit requested, quitting...') - break - - self.shm.close() - - -class FileWorker(Process): - def __init__(self, queue, out_queue, base_path, shm, cache_path=None, logging_queue=None): - super().__init__(name='FileWorker') - self.q = queue - self.o_q = out_queue - self.base_path = base_path - self.cache_path = cache_path if cache_path else os.path.join(base_path, '.cache') - self.shm = SharedMemory(name=shm) - self.log_level = logging.getLogger().level - self.logging_queue = logging_queue - - def run(self): - # we have to fix up the logger before we can start - _root = logging.getLogger() - _root.handlers = [] - _root.addHandler(QueueHandler(self.logging_queue)) - - logger = logging.getLogger(self.name) - logger.setLevel(self.log_level) - logger.debug(f'Download worker reporting for duty!') - - last_filename = '' - current_file = None - - while True: - try: - try: - j = self.q.get(timeout=10.0) - except Empty: - logger.warning('Writer queue empty!') - continue - - if j.kill: - if current_file: - current_file.close() - self.o_q.put(WriterTaskResult(success=True, kill=True)) - break - - # make directories if required - path = os.path.split(j.filename)[0] - if not os.path.exists(os.path.join(self.base_path, path)): - os.makedirs(os.path.join(self.base_path, path)) - - full_path = os.path.join(self.base_path, j.filename) - - if j.empty: # just create an empty file - open(full_path, 'a').close() - self.o_q.put(WriterTaskResult(success=True, filename=j.filename)) - continue - elif j.open: - if current_file: - logger.warning(f'Opening new file {j.filename} without closing previous! {last_filename}') - current_file.close() - - current_file = open(full_path, 'wb') - last_filename = j.filename - - self.o_q.put(WriterTaskResult(success=True, filename=j.filename)) - continue - elif j.close: - if current_file: - current_file.close() - current_file = None - else: - logger.warning(f'Asking to close file that is not open: {j.filename}') - - self.o_q.put(WriterTaskResult(success=True, filename=j.filename, closed=True)) - continue - elif j.rename: - if current_file: - logger.warning('Trying to rename file without closing first!') - current_file.close() - current_file = None - if j.delete: - try: - os.remove(full_path) - except OSError as e: - logger.error(f'Removing file failed: {e!r}') - self.o_q.put(WriterTaskResult(success=False, filename=j.filename)) - continue - - try: - os.rename(os.path.join(self.base_path, j.old_filename), full_path) - except OSError as e: - logger.error(f'Renaming file failed: {e!r}') - self.o_q.put(WriterTaskResult(success=False, filename=j.filename)) - continue - - self.o_q.put(WriterTaskResult(success=True, filename=j.filename)) - continue - elif j.delete: - if current_file: - logger.warning('Trying to delete file without closing first!') - current_file.close() - current_file = None - - try: - os.remove(full_path) - except OSError as e: - if not j.silent: - logger.error(f'Removing file failed: {e!r}') - - self.o_q.put(WriterTaskResult(success=True, filename=j.filename)) - continue - - pre_write = post_write = 0 - - try: - if j.shm: - pre_write = time.time() - shm_offset = j.shm.offset + j.chunk_offset - shm_end = shm_offset + j.chunk_size - current_file.write(self.shm.buf[shm_offset:shm_end].tobytes()) - post_write = time.time() - elif j.cache_file: - pre_write = time.time() - with open(os.path.join(self.cache_path, j.cache_file), 'rb') as f: - if j.chunk_offset: - f.seek(j.chunk_offset) - current_file.write(f.read(j.chunk_size)) - post_write = time.time() - elif j.old_file: - pre_write = time.time() - with open(os.path.join(self.base_path, j.old_file), 'rb') as f: - if j.chunk_offset: - f.seek(j.chunk_offset) - current_file.write(f.read(j.chunk_size)) - post_write = time.time() - except Exception as e: - logger.warning(f'Something in writing a file failed: {e!r}') - self.o_q.put(WriterTaskResult(success=False, filename=j.filename, - chunk_guid=j.chunk_guid, - release_memory=j.release_memory, - shm=j.shm, size=j.chunk_size, - time_delta=post_write - pre_write)) - else: - self.o_q.put(WriterTaskResult(success=True, filename=j.filename, - chunk_guid=j.chunk_guid, - release_memory=j.release_memory, - shm=j.shm, size=j.chunk_size, - time_delta=post_write - pre_write)) - except Exception as e: - logger.warning(f'Job {j.filename} failed with: {e!r}, fetching next one...') - self.o_q.put(WriterTaskResult(success=False, filename=j.filename, chunk_guid=j.chunk_guid)) - - try: - if current_file: - current_file.close() - current_file = None - except Exception as e: - logger.error(f'Closing file after error failed: {e!r}') - except KeyboardInterrupt: - logger.warning('Immediate exit requested, quitting...') - if current_file: - current_file.close() - return diff --git a/custom_legendary/lfs/__init__.py b/custom_legendary/lfs/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/custom_legendary/lfs/egl.py b/custom_legendary/lfs/egl.py deleted file mode 100644 index 67693f2b..00000000 --- a/custom_legendary/lfs/egl.py +++ /dev/null @@ -1,81 +0,0 @@ -# coding: utf-8 - -import configparser -import json -import os -from typing import List - -from custom_legendary.models.egl import EGLManifest - - -class EPCLFS: - def __init__(self): - if os.name == 'nt': - self.appdata_path = os.path.expandvars( - r'%LOCALAPPDATA%\EpicGamesLauncher\Saved\Config\Windows' - ) - self.programdata_path = os.path.expandvars( - r'%PROGRAMDATA%\Epic\EpicGamesLauncher\Data\Manifests' - ) - else: - self.appdata_path = self.programdata_path = None - - self.config = configparser.ConfigParser(strict=False) - self.config.optionxform = lambda option: option - - self.manifests = dict() - - def read_config(self): - if not self.appdata_path: - raise ValueError('EGS AppData path is not set') - - self.config.read(os.path.join(self.appdata_path, 'GameUserSettings.ini')) - - def save_config(self): - if not self.appdata_path: - raise ValueError('EGS AppData path is not set') - - with open(os.path.join(self.appdata_path, 'GameUserSettings.ini'), 'w') as f: - self.config.write(f, space_around_delimiters=False) - - def read_manifests(self): - if not self.programdata_path: - raise ValueError('EGS ProgramData path is not set') - - for f in os.listdir(self.programdata_path): - if f.endswith('.item'): - data = json.load(open(os.path.join(self.programdata_path, f))) - self.manifests[data['AppName']] = data - - def get_manifests(self) -> List[EGLManifest]: - if not self.manifests: - self.read_manifests() - - return [EGLManifest.from_json(m) for m in self.manifests.values()] - - def get_manifest(self, app_name) -> EGLManifest: - if not self.manifests: - self.read_manifests() - - if app_name in self.manifests: - return EGLManifest.from_json(self.manifests[app_name]) - else: - raise ValueError('Cannot find manifest') - - def set_manifest(self, manifest: EGLManifest): - if not self.programdata_path: - raise ValueError('EGS ProgramData path is not set') - - manifest_data = manifest.to_json() - self.manifests[manifest.app_name] = manifest_data - with open(os.path.join(self.programdata_path, f'{manifest.installation_guid}.item'), 'w') as f: - json.dump(manifest_data, f, indent=4, sort_keys=True) - - def delete_manifest(self, app_name): - if not self.manifests: - self.read_manifests() - if app_name not in self.manifests: - raise ValueError('AppName is not in manifests!') - - manifest = EGLManifest.from_json(self.manifests.pop(app_name)) - os.remove(os.path.join(self.programdata_path, f'{manifest.installation_guid}.item')) diff --git a/custom_legendary/lfs/lgndry.py b/custom_legendary/lfs/lgndry.py deleted file mode 100644 index 70ba16d4..00000000 --- a/custom_legendary/lfs/lgndry.py +++ /dev/null @@ -1,267 +0,0 @@ -# coding: utf-8 - -import configparser -import json -import logging -import os -from pathlib import Path - -from custom_legendary.models.game import * -from custom_legendary.utils.lfs import clean_filename - - -class LGDLFS: - def __init__(self): - self.log = logging.getLogger('LGDLFS') - - if config_path := os.environ.get('XDG_CONFIG_HOME'): - self.path = os.path.join(config_path, 'legendary') - else: - self.path = os.path.expanduser('~/.config/legendary') - - # EGS user info - self._user_data = None - # EGS entitlements - self._entitlements = None - # EGS asset data - self._assets = None - # EGS metadata - self._game_metadata = dict() - # Config with game specific settings (e.g. start parameters, env variables) - self.config = configparser.ConfigParser(comment_prefixes='/', allow_no_value=True) - self.config.optionxform = str - - # ensure folders exist. - for f in ['', 'manifests', 'metadata', 'tmp']: - if not os.path.exists(os.path.join(self.path, f)): - os.makedirs(os.path.join(self.path, f)) - - # if "old" folder exists migrate files and remove it - if os.path.exists(os.path.join(self.path, 'manifests', 'old')): - self.log.info('Migrating manifest files from old folders to new, please wait...') - # remove unversioned manifest files - for _f in os.listdir(os.path.join(self.path, 'manifests')): - if '.manifest' not in _f: - continue - if '_' not in _f or (_f.startswith('UE_') and _f.count('_') < 2): - self.log.debug(f'Deleting "{_f}" ...') - os.remove(os.path.join(self.path, 'manifests', _f)) - - # move files from "old" to the base folder - for _f in os.listdir(os.path.join(self.path, 'manifests', 'old')): - try: - self.log.debug(f'Renaming "{_f}"') - os.rename(os.path.join(self.path, 'manifests', 'old', _f), - os.path.join(self.path, 'manifests', _f)) - except Exception as e: - self.log.warning(f'Renaming manifest file "{_f}" failed: {e!r}') - - # remove "old" folder - try: - os.removedirs(os.path.join(self.path, 'manifests', 'old')) - except Exception as e: - self.log.warning(f'Removing "{os.path.join(self.path, "manifests", "old")}" folder failed: ' - f'{e!r}, please remove manually') - - # try loading config - self.config.read(os.path.join(self.path, 'config.ini')) - # make sure "Legendary" section exists - if 'Legendary' not in self.config: - self.config['Legendary'] = dict() - - try: - self._installed = json.load(open(os.path.join(self.path, 'installed.json'))) - except Exception as e: # todo do not do this - self._installed = None - - # load existing app metadata - for gm_file in os.listdir(os.path.join(self.path, 'metadata')): - try: - _meta = json.load(open(os.path.join(self.path, 'metadata', gm_file))) - self._game_metadata[_meta['app_name']] = _meta - except Exception as e: - self.log.debug(f'Loading game meta file "{gm_file}" failed: {e!r}') - - @property - def userdata(self): - if self._user_data is not None: - return self._user_data - - try: - self._user_data = json.load(open(os.path.join(self.path, 'user.json'))) - return self._user_data - except Exception as e: - self.log.debug(f'Failed to load user data: {e!r}') - return None - - @userdata.setter - def userdata(self, userdata): - if userdata is None: - raise ValueError('Userdata is none!') - - self._user_data = userdata - json.dump(userdata, open(os.path.join(self.path, 'user.json'), 'w'), - indent=2, sort_keys=True) - - def invalidate_userdata(self): - self._user_data = None - if os.path.exists(os.path.join(self.path, 'user.json')): - os.remove(os.path.join(self.path, 'user.json')) - - @property - def entitlements(self): - if self._entitlements is not None: - return self._entitlements - - try: - self._entitlements = json.load(open(os.path.join(self.path, 'entitlements.json'))) - return self._entitlements - except Exception as e: - self.log.debug(f'Failed to load entitlements data: {e!r}') - return None - - @entitlements.setter - def entitlements(self, entitlements): - if entitlements is None: - raise ValueError('Entitlements is none!') - - self._entitlements = entitlements - json.dump(entitlements, open(os.path.join(self.path, 'entitlements.json'), 'w'), - indent=2, sort_keys=True) - - @property - def assets(self): - if self._assets is None: - try: - self._assets = [GameAsset.from_json(a) for a in - json.load(open(os.path.join(self.path, 'assets.json')))] - except Exception as e: - self.log.debug(f'Failed to load assets data: {e!r}') - return None - - return self._assets - - @assets.setter - def assets(self, assets): - if assets is None: - raise ValueError('Assets is none!') - - self._assets = assets - json.dump([a.__dict__ for a in self._assets], - open(os.path.join(self.path, 'assets.json'), 'w'), - indent=2, sort_keys=True) - - def _get_manifest_filename(self, app_name, version): - fname = clean_filename(f'{app_name}_{version}') - return os.path.join(self.path, 'manifests', f'{fname}.manifest') - - def load_manifest(self, app_name, version): - try: - return open(self._get_manifest_filename(app_name, version), 'rb').read() - except FileNotFoundError: # all other errors should propagate - return None - - def save_manifest(self, app_name, manifest_data, version): - with open(self._get_manifest_filename(app_name, version), 'wb') as f: - f.write(manifest_data) - - def get_game_meta(self, app_name): - _meta = self._game_metadata.get(app_name, None) - if _meta: - return Game.from_json(_meta) - return None - - def set_game_meta(self, app_name, meta): - json_meta = meta.__dict__ - self._game_metadata[app_name] = json_meta - meta_file = os.path.join(self.path, 'metadata', f'{app_name}.json') - json.dump(json_meta, open(meta_file, 'w'), indent=2, sort_keys=True) - - def delete_game_meta(self, app_name): - if app_name in self._game_metadata: - del self._game_metadata[app_name] - meta_file = os.path.join(self.path, 'metadata', f'{app_name}.json') - if os.path.exists(meta_file): - os.remove(meta_file) - else: - raise ValueError(f'Game {app_name} does not exist in metadata DB!') - - def get_tmp_path(self): - return os.path.join(self.path, 'tmp') - - def clean_tmp_data(self): - for f in os.listdir(os.path.join(self.path, 'tmp')): - try: - os.remove(os.path.join(self.path, 'tmp', f)) - except Exception as e: - self.log.warning(f'Failed to delete file "{f}": {e!r}') - - def clean_metadata(self, app_names): - for f in os.listdir(os.path.join(self.path, 'metadata')): - app_name = f.rpartition('.')[0] - if app_name not in app_names: - try: - os.remove(os.path.join(self.path, 'metadata', f)) - except Exception as e: - self.log.warning(f'Failed to delete file "{f}": {e!r}') - - def clean_manifests(self, in_use): - in_use_files = set(f'{clean_filename(f"{app_name}_{version}")}.manifest' for app_name, version in in_use) - for f in os.listdir(os.path.join(self.path, 'manifests')): - if f not in in_use_files: - try: - os.remove(os.path.join(self.path, 'manifests', f)) - except Exception as e: - self.log.warning(f'Failed to delete file "{f}": {e!r}') - - def get_installed_game(self, app_name): - if self._installed is None: - try: - self._installed = json.load(open(os.path.join(self.path, 'installed.json'))) - except Exception as e: - self.log.debug(f'Failed to load installed game data: {e!r}') - return None - - game_json = self._installed.get(app_name, None) - if game_json: - return InstalledGame.from_json(game_json) - return None - - def set_installed_game(self, app_name, install_info): - if self._installed is None: - self._installed = dict() - - if app_name in self._installed: - self._installed[app_name].update(install_info.__dict__) - else: - self._installed[app_name] = install_info.__dict__ - - json.dump(self._installed, open(os.path.join(self.path, 'installed.json'), 'w'), - indent=2, sort_keys=True) - - def remove_installed_game(self, app_name): - if self._installed is None: - self.log.warning('Trying to remove a game, but no installed games?!') - return - - if app_name in self._installed: - del self._installed[app_name] - else: - self.log.warning('Trying to remove non-installed game:', app_name) - return - - json.dump(self._installed, open(os.path.join(self.path, 'installed.json'), 'w'), - indent=2, sort_keys=True) - - def get_installed_list(self): - if not self._installed: - return [] - - return [InstalledGame.from_json(i) for i in self._installed.values()] - - def save_config(self): - with open(os.path.join(self.path, 'config.ini'), 'w') as cf: - self.config.write(cf) - - def get_dir_size(self): - return sum(f.stat().st_size for f in Path(self.path).glob('**/*') if f.is_file()) diff --git a/custom_legendary/models/__init__.py b/custom_legendary/models/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/custom_legendary/models/chunk.py b/custom_legendary/models/chunk.py deleted file mode 100644 index d7605942..00000000 --- a/custom_legendary/models/chunk.py +++ /dev/null @@ -1,149 +0,0 @@ -# coding: utf-8 - -import struct -import zlib -from hashlib import sha1 -from io import BytesIO -from uuid import uuid4 - -from custom_legendary.utils.rolling_hash import get_hash - - -# ToDo do some reworking to make this more memory efficient -class Chunk: - header_magic = 0xB1FE3AA2 - - def __init__(self): - self.header_version = 3 - self.header_size = 0 - self.compressed_size = 0 - self.hash = 0 - self.stored_as = 0 - self.guid = struct.unpack('>IIII', uuid4().bytes) - - # 0x1 = rolling hash, 0x2 = sha hash, 0x3 = both - self.hash_type = 0 - self.sha_hash = None - self.uncompressed_size = 1024 * 1024 - - self._guid_str = '' - self._guid_num = 0 - self._bio = None - self._data = None - - @property - def data(self): - if self._data: - return self._data - - if self.compressed: - self._data = zlib.decompress(self._bio.read()) - else: - self._data = self._bio.read() - - # close BytesIO with raw data since we no longer need it - self._bio.close() - self._bio = None - - return self._data - - @data.setter - def data(self, value: bytes): - if len(value) > 1024 * 1024: - raise ValueError('Provided data is too large (> 1 MiB)!') - # data is now uncompressed - if self.compressed: - self.stored_as ^= 0x1 - # pad data to 1 MiB - if len(value) < 1024 * 1024: - value += b'\x00' * (1024 * 1024 - len(value)) - # recalculate hashes - self.hash = get_hash(value) - self.sha_hash = sha1(value).digest() - self.hash_type = 0x3 - self._data = value - - @property - def guid_str(self): - if not self._guid_str: - self._guid_str = '-'.join('{:08x}'.format(g) for g in self.guid) - return self._guid_str - - @property - def guid_num(self): - if not self._guid_num: - self._guid_num = self.guid[3] + (self.guid[2] << 32) + (self.guid[1] << 64) + (self.guid[0] << 96) - return self._guid_num - - @property - def compressed(self): - return self.stored_as & 0x1 - - @classmethod - def read_buffer(cls, data): - _sio = BytesIO(data) - return cls.read(_sio) - - @classmethod - def read(cls, bio): - head_start = bio.tell() - - if struct.unpack('= 2: - _chunk.sha_hash = bio.read(20) - _chunk.hash_type = struct.unpack('B', bio.read(1))[0] - - if _chunk.header_version >= 3: - _chunk.uncompressed_size = struct.unpack(' dict: - out = _template.copy() - out.update(self.remainder) - out['AppName'] = self.app_name - out['AppVersionString'] = self.app_version_string - out['BaseURLs'] = self.base_urls - out['BuildLabel'] = self.build_label - out['CatalogItemId'] = self.catalog_item_id - out['CatalogNamespace'] = self.namespace - out['DisplayName'] = self.display_name - out['InstallLocation'] = self.install_location - out['InstallSize'] = self.install_size - out['InstallTags'] = self.install_tags - out['InstallationGuid'] = self.installation_guid - out['LaunchCommand'] = self.launch_command - out['LaunchExecutable'] = self.executable - out['MainGameAppName'] = self.main_game_appname - out['MandatoryAppFolderName'] = self.app_folder_name - out['ManifestLocation'] = self.manifest_location - out['OwnershipToken'] = str(self.ownership_token).lower() - out['StagingLocation'] = self.staging_location - out['bCanRunOffline'] = self.can_run_offline - out['bIsIncompleteInstall'] = self.is_incomplete_install - out['bNeedsValidation'] = self.needs_validation - return out - - @classmethod - def from_lgd_game(cls, game: Game, igame: InstalledGame): - tmp = cls() - tmp.app_name = game.app_name - tmp.app_version_string = igame.version - tmp.base_urls = igame.base_urls - tmp.build_label = 'Live' - tmp.catalog_item_id = game.asset_info.catalog_item_id - tmp.namespace = game.asset_info.namespace - tmp.display_name = igame.title - tmp.install_location = igame.install_path - tmp.install_size = igame.install_size - tmp.install_tags = igame.install_tags - tmp.installation_guid = igame.egl_guid - tmp.launch_command = igame.launch_parameters - tmp.executable = igame.executable - tmp.main_game_appname = game.app_name # todo for DLC support this needs to be the base game - tmp.app_folder_name = game.metadata.get('customAttributes', {}).get('FolderName', {}).get('value', '') - tmp.manifest_location = igame.install_path + '/.egstore' - tmp.ownership_token = igame.requires_ot - tmp.staging_location = igame.install_path + '/.egstore/bps' - tmp.can_run_offline = igame.can_run_offline - tmp.is_incomplete_install = False - tmp.needs_validation = igame.needs_verification - return tmp - - def to_lgd_igame(self) -> InstalledGame: - return InstalledGame(app_name=self.app_name, title=self.display_name, version=self.app_version_string, - base_urls=self.base_urls, install_path=self.install_location, executable=self.executable, - launch_parameters=self.launch_command, can_run_offline=self.can_run_offline, - requires_ot=self.ownership_token, is_dlc=False, - needs_verification=self.needs_validation, install_size=self.install_size, - egl_guid=self.installation_guid, install_tags=self.install_tags) diff --git a/custom_legendary/models/exceptions.py b/custom_legendary/models/exceptions.py deleted file mode 100644 index 9f269312..00000000 --- a/custom_legendary/models/exceptions.py +++ /dev/null @@ -1,12 +0,0 @@ -# coding: utf-8 - -# ToDo more custom exceptions where it makes sense - - -class CaptchaError(Exception): - """Raised by core if direct login fails""" - pass - - -class InvalidCredentialsError(Exception): - pass diff --git a/custom_legendary/models/game.py b/custom_legendary/models/game.py deleted file mode 100644 index b2ea5682..00000000 --- a/custom_legendary/models/game.py +++ /dev/null @@ -1,182 +0,0 @@ -# coding: utf-8 - -from enum import Enum - - -class GameAsset: - def __init__(self): - self.app_name = '' - self.asset_id = '' - self.build_version = '' - self.catalog_item_id = '' - self.label_name = '' - self.namespace = '' - self.metadata = dict() - - @classmethod - def from_egs_json(cls, json): - tmp = cls() - tmp.app_name = json.get('appName', '') - tmp.asset_id = json.get('assetId', '') - tmp.build_version = json.get('buildVersion', '') - tmp.catalog_item_id = json.get('catalogItemId', '') - tmp.label_name = json.get('labelName', '') - tmp.namespace = json.get('namespace', '') - tmp.metadata = json.get('metadata', {}) - return tmp - - @classmethod - def from_json(cls, json): - tmp = cls() - tmp.app_name = json.get('app_name', '') - tmp.asset_id = json.get('asset_id', '') - tmp.build_version = json.get('build_version', '') - tmp.catalog_item_id = json.get('catalog_item_id', '') - tmp.label_name = json.get('label_name', '') - tmp.namespace = json.get('namespace', '') - tmp.metadata = json.get('metadata', {}) - return tmp - - -class Game: - def __init__(self, app_name='', app_title='', asset_info=None, app_version='', metadata=None): - self.metadata = dict() if metadata is None else metadata # store metadata from EGS - self.asset_info = asset_info if asset_info else GameAsset() # asset info from EGS - - self.app_version = app_version - self.app_name = app_name - self.app_title = app_title - self.base_urls = [] # base urls for download, only really used when cached manifest is current - - @property - def is_dlc(self): - return self.metadata and 'mainGameItem' in self.metadata - - @property - def supports_cloud_saves(self): - return self.metadata and (self.metadata.get('customAttributes', {}).get('CloudSaveFolder') is not None) - - @classmethod - def from_json(cls, json): - tmp = cls() - tmp.metadata = json.get('metadata', dict()) - tmp.asset_info = GameAsset.from_json(json.get('asset_info', dict())) - tmp.app_name = json.get('app_name', 'undefined') - tmp.app_title = json.get('app_title', 'undefined') - tmp.app_version = json.get('app_version', 'undefined') - tmp.base_urls = json.get('base_urls', list()) - return tmp - - @property - def __dict__(self): - """This is just here so asset_info gets turned into a dict as well""" - return dict(metadata=self.metadata, asset_info=self.asset_info.__dict__, - app_name=self.app_name, app_title=self.app_title, - app_version=self.app_version, base_urls=self.base_urls) - - -class InstalledGame: - def __init__(self, app_name='', title='', version='', manifest_path='', base_urls=None, - install_path='', executable='', launch_parameters='', prereq_info=None, - can_run_offline=False, requires_ot=False, is_dlc=False, save_path=None, - needs_verification=False, install_size=0, egl_guid='', install_tags=None): - self.app_name = app_name - self.title = title - self.version = version - - self.manifest_path = manifest_path - self.base_urls = list() if not base_urls else base_urls - self.install_path = install_path - self.executable = executable - self.launch_parameters = launch_parameters - self.prereq_info = prereq_info - self.can_run_offline = can_run_offline - self.requires_ot = requires_ot - self.is_dlc = is_dlc - self.save_path = save_path - self.needs_verification = needs_verification - self.install_size = install_size - self.egl_guid = egl_guid - self.install_tags = install_tags if install_tags else [] - - @classmethod - def from_json(cls, json): - tmp = cls() - tmp.app_name = json.get('app_name', '') - tmp.version = json.get('version', '') - tmp.title = json.get('title', '') - - tmp.manifest_path = json.get('manifest_path', '') - tmp.base_urls = json.get('base_urls', list()) - tmp.install_path = json.get('install_path', '') - tmp.executable = json.get('executable', '') - tmp.launch_parameters = json.get('launch_parameters', '') - tmp.prereq_info = json.get('prereq_info', None) - - tmp.can_run_offline = json.get('can_run_offline', False) - tmp.requires_ot = json.get('requires_ot', False) - tmp.is_dlc = json.get('is_dlc', False) - tmp.save_path = json.get('save_path', None) - tmp.needs_verification = json.get('needs_verification', False) is True - tmp.install_size = json.get('install_size', 0) - tmp.egl_guid = json.get('egl_guid', '') - tmp.install_tags = json.get('install_tags', []) - return tmp - - -class SaveGameFile: - def __init__(self, app_name='', filename='', manifest='', datetime=None): - self.app_name = app_name - self.filename = filename - self.manifest_name = manifest - self.datetime = datetime - - -class SaveGameStatus(Enum): - LOCAL_NEWER = 0 - REMOTE_NEWER = 1 - SAME_AGE = 2 - NO_SAVE = 3 - - -class VerifyResult(Enum): - HASH_MATCH = 0 - HASH_MISMATCH = 1 - FILE_MISSING = 2 - OTHER_ERROR = 3 - - -x = {'title': 'Frostpunk', - 'id': 'b43c1e1e0ca14b6784b323c59c751136', 'namespace': 'd5241c76f178492ea1540fce45616757', - 'description': 'Frostpunk', 'effectiveDate': '2099-01-01T00:00:00.000Z', 'offerType': 'OTHERS', 'expiryDate': None, - 'status': 'ACTIVE', 'isCodeRedemptionOnly': True, 'keyImages': [{'type': 'VaultClosed', - 'url': 'https://cdn1.epicgames.com/d5241c76f178492ea1540fce45616757/offer/EpicVault_Clean_OPEN_V10_LightsON-1920x1080-75e6d0636a6083944570a1c6f94ead4f.png'}, - {'type': 'DieselStoreFrontWide', - 'url': 'https://cdn1.epicgames.com/salesEvent/salesEvent/EGS_Frostpunk_wide_2560x1440-ef2f4d458120af0839dde35b1a022828'}, - {'type': 'DieselStoreFrontTall', - 'url': 'https://cdn1.epicgames.com/salesEvent/salesEvent/EGS_Frostpunk_Tall_1200x1600-c71dc27cfe505c6c662c49011b36a0c5'}], - 'seller': {'id': 'o-ufmrk5furrrxgsp5tdngefzt5rxdcn', 'name': 'Epic Dev Test Account'}, 'productSlug': 'frostpunk', - 'urlSlug': 'free-games-06', 'url': None, - 'items': [{'id': '8341d7c7e4534db7848cc428aa4cbe5a', 'namespace': 'd5241c76f178492ea1540fce45616757'}], - 'customAttributes': [{'key': 'com.epicgames.app.freegames.vault.close', 'value': '[]'}, - {'key': 'com.epicgames.app.blacklist', 'value': '[]'}, - {'key': 'com.epicgames.app.freegames.vault.slug', - 'value': 'news/the-epic-mega-sale-returns-for-2021'}, - {'key': 'publisherName', 'value': '11 bit studios'}, {'key': 'dupe', 'value': '[]'}, - {'key': 'com.epicgames.app.freegames.vault.open', 'value': '[]'}, - {'key': 'developerName', 'value': '11 bit studios'}, - {'key': 'com.epicgames.app.productSlug', 'value': 'frostpunk'}], - 'categories': [{'path': 'freegames/vaulted'}, {'path': 'freegames'}, {'path': 'games'}, {'path': 'applications'}], - 'tags': [], 'price': {'totalPrice': {'discountPrice': 0, 'originalPrice': 0, 'voucherDiscount': 0, 'discount': 0, - 'currencyCode': 'USD', 'currencyInfo': {'decimals': 2}, - 'fmtPrice': {'originalPrice': '0', 'discountPrice': '0', - 'intermediatePrice': '0'}}, - 'lineOffers': [{'appliedRules': []}]}, 'promotions': {'promotionalOffers': [], - 'upcomingPromotionalOffers': [{ - 'promotionalOffers': [ - { - 'startDate': '2021-06-03T15:00:00.000Z', - 'endDate': '2021-06-10T15:00:00.000Z', - 'discountSetting': { - 'discountType': 'PERCENTAGE', - 'discountPercentage': 0}}]}]}} diff --git a/custom_legendary/models/json_manifest.py b/custom_legendary/models/json_manifest.py deleted file mode 100644 index 24f01c23..00000000 --- a/custom_legendary/models/json_manifest.py +++ /dev/null @@ -1,178 +0,0 @@ -# coding: utf-8 - -import json -import struct -from copy import deepcopy - -from custom_legendary.models.manifest import ( - Manifest, ManifestMeta, CDL, ChunkPart, ChunkInfo, FML, FileManifest, CustomFields -) - - -def blob_to_num(in_str): - """ - The JSON manifest use a rather strange format for storing numbers. - - It's essentially %03d for each char concatenated to a string. - ...instead of just putting the fucking number in the JSON... - - Also it's still little endian so we have to bitshift it. - - """ - num = 0 - shift = 0 - for i in range(0, len(in_str), 3): - num += (int(in_str[i:i + 3]) << shift) - shift += 8 - return num - - -def guid_from_json(in_str): - return struct.unpack('>IIII', bytes.fromhex(in_str)) - - -class JSONManifest(Manifest): - """ - Manifest-compatible reader for JSON based manifests - - """ - - def __init__(self): - super().__init__() - self.json_data = None - - @classmethod - def read_all(cls, manifest): - _m = cls.read(manifest) - _tmp = deepcopy(_m.json_data) - - _m.meta = JSONManifestMeta.read(_tmp) - _m.chunk_data_list = JSONCDL.read(_tmp, manifest_version=_m.version) - _m.file_manifest_list = JSONFML.read(_tmp) - _m.custom_fields = CustomFields() - _m.custom_fields._dict = _tmp.pop('CustomFields', dict()) - - if _tmp.keys(): - print(f'Did not read JSON keys: {_tmp.keys()}!') - - # clear raw data after manifest has been loaded - _m.data = b'' - _m.json_data = None - - return _m - - @classmethod - def read(cls, manifest): - _manifest = cls() - _manifest.data = manifest - _manifest.json_data = json.loads(manifest.decode('utf-8')) - - _manifest.stored_as = 0 # never compressed - _manifest.version = blob_to_num(_manifest.json_data.get('ManifestFileVersion', '013000000000')) - - return _manifest - - def write(self, *args, **kwargs): - # The version here only matters for the manifest header, - # the feature level in meta determines chunk folders etc. - # So all that's required for successful serialization is - # setting it to something high enough to be a binary manifest - self.version = 18 - return super().write(*args, **kwargs) - - -class JSONManifestMeta(ManifestMeta): - def __init__(self): - super().__init__() - - @classmethod - def read(cls, json_data): - _meta = cls() - - _meta.feature_level = blob_to_num(json_data.pop('ManifestFileVersion', '013000000000')) - _meta.is_file_data = json_data.pop('bIsFileData', False) - _meta.app_id = blob_to_num(json_data.pop('AppID', '000000000000')) - _meta.app_name = json_data.pop('AppNameString', '') - _meta.build_version = json_data.pop('BuildVersionString', '') - _meta.launch_exe = json_data.pop('LaunchExeString', '') - _meta.launch_command = json_data.pop('LaunchCommand', '') - _meta.prereq_ids = json_data.pop('PrereqIds', list()) - _meta.prereq_name = json_data.pop('PrereqName', '') - _meta.prereq_path = json_data.pop('PrereqPath', '') - _meta.prereq_args = json_data.pop('PrereqArgs', '') - - return _meta - - -class JSONCDL(CDL): - def __init__(self): - super().__init__() - - @classmethod - def read(cls, json_data, manifest_version=13): - _cdl = cls() - _cdl._manifest_version = manifest_version - _cdl.count = len(json_data['ChunkFilesizeList']) - - cfl = json_data.pop('ChunkFilesizeList') - chl = json_data.pop('ChunkHashList') - csl = json_data.pop('ChunkShaList') - dgl = json_data.pop('DataGroupList') - _guids = list(cfl.keys()) - - for guid in _guids: - _ci = ChunkInfo(manifest_version=manifest_version) - _ci.guid = guid_from_json(guid) - _ci.file_size = blob_to_num(cfl.pop(guid)) - _ci.hash = blob_to_num(chl.pop(guid)) - _ci.sha_hash = bytes.fromhex(csl.pop(guid)) - _ci.group_num = blob_to_num(dgl.pop(guid)) - _ci.window_size = 1024 * 1024 - _cdl.elements.append(_ci) - - for _dc in (cfl, chl, csl, dgl): - if _dc: - print(f'Non-consumed CDL stuff: {_dc}') - - return _cdl - - -class JSONFML(FML): - def __init__(self): - super().__init__() - - @classmethod - def read(cls, json_data): - _fml = cls() - _fml.count = len(json_data['FileManifestList']) - - for _fmj in json_data.pop('FileManifestList'): - _fm = FileManifest() - _fm.filename = _fmj.pop('Filename', '') - _fm.hash = blob_to_num(_fmj.pop('FileHash')).to_bytes(160 // 8, 'little') - _fm.flags |= int(_fmj.pop('bIsReadOnly', False)) - _fm.flags |= int(_fmj.pop('bIsCompressed', False)) << 1 - _fm.flags |= int(_fmj.pop('bIsUnixExecutable', False)) << 2 - _fm.file_size = 0 - _fm.chunk_parts = [] - _fm.install_tags = _fmj.pop('InstallTags', list()) - - _offset = 0 - for _cpj in _fmj.pop('FileChunkParts'): - _cp = ChunkPart() - _cp.guid = guid_from_json(_cpj.pop('Guid')) - _cp.offset = blob_to_num(_cpj.pop('Offset')) - _cp.size = blob_to_num(_cpj.pop('Size')) - _cp.file_offset = _offset - _fm.file_size += _cp.size - if _cpj: - print(f'Non-read ChunkPart keys: {_cpj.keys()}') - _fm.chunk_parts.append(_cp) - _offset += _cp.size - - if _fmj: - print(f'Non-read FileManifest keys: {_fmj.keys()}') - - _fml.elements.append(_fm) - - return _fml diff --git a/custom_legendary/models/manifest.py b/custom_legendary/models/manifest.py deleted file mode 100644 index 52fe4433..00000000 --- a/custom_legendary/models/manifest.py +++ /dev/null @@ -1,739 +0,0 @@ -# coding: utf-8 - -import hashlib -import logging -import struct -import zlib - -from base64 import b64encode -from io import BytesIO - -logger = logging.getLogger('Manifest') - - -def read_fstring(bio): - length = struct.unpack(' 0: - s = bio.read(length - 1).decode('ascii') - bio.seek(1, 1) # skip string null terminator - else: # empty string, no terminators or anything - s = '' - - return s - - -def write_fstring(bio, string): - if not string: - bio.write(struct.pack('= 15: - return 'ChunksV4' - elif version >= 6: - return 'ChunksV3' - elif version >= 3: - return 'ChunksV2' - else: - return 'Chunks' - - -class Manifest: - header_magic = 0x44BEC00C - - def __init__(self): - self.header_size = 41 - self.size_compressed = 0 - self.size_uncompressed = 0 - self.sha_hash = '' - self.stored_as = 0 - self.version = 18 - self.data = b'' - - # remainder - self.meta = None - self.chunk_data_list = None - self.file_manifest_list = None - self.custom_fields = None - - @property - def compressed(self): - return self.stored_as & 0x1 - - @classmethod - def read_all(cls, data): - _m = cls.read(data) - _tmp = BytesIO(_m.data) - - _m.meta = ManifestMeta.read(_tmp) - _m.chunk_data_list = CDL.read(_tmp, _m.meta.feature_level) - _m.file_manifest_list = FML.read(_tmp) - _m.custom_fields = CustomFields.read(_tmp) - - unhandled_data = _tmp.read() - if unhandled_data: - logger.warning(f'Did not read {len(unhandled_data)} remaining bytes in manifest! ' - f'This may not be a problem.') - - # Throw this away since the raw data is no longer needed - _tmp.close() - del _tmp - _m.data = b'' - - return _m - - @classmethod - def read(cls, data): - bio = BytesIO(data) - if struct.unpack(' 0: - _meta._build_id = read_fstring(bio) - - if bio.tell() != _meta.meta_size: - raise ValueError('Did not read entire meta!') - - return _meta - - def write(self, bio): - meta_start = bio.tell() - - bio.write(struct.pack(' 0: - write_fstring(bio, self.build_id) - - meta_end = bio.tell() - bio.seek(meta_start) - bio.write(struct.pack(''.format( - self.guid_str, self.hash, self.sha_hash.hex(), self.group_num, self.window_size, self.file_size - ) - - @property - def guid_str(self): - if not self._guid_str: - self._guid_str = '-'.join('{:08x}'.format(g) for g in self.guid) - - return self._guid_str - - @property - def guid_num(self): - if not self._guid_num: - self._guid_num = self.guid[3] + (self.guid[2] << 32) + (self.guid[1] << 64) + (self.guid[0] << 96) - return self._guid_num - - @property - def group_num(self): - if self._guid_num is not None: - return self._group_num - - self._group_num = (zlib.crc32( - struct.pack(' 0: - logger.warning(f'Did not read {diff} bytes from chunk part!') - bio.seek(diff) - - # we have to calculate the actual file size ourselves - for fm in _fml.elements: - fm.file_size = sum(c.size for c in fm.chunk_parts) - - if bio.tell() - fml_start != _fml.size: - raise ValueError('Did not read entire chunk data list!') - - return _fml - - def write(self, bio): - fml_start = bio.tell() - bio.write(struct.pack(''.format( - self.filename, self.symlink_target, self.hash.hex(), self.flags, - ', '.join(self.install_tags), cp_repr, self.file_size - ) - - -class ChunkPart: - def __init__(self, guid=None, offset=0, size=0, file_offset=0): - self.guid = guid - self.offset = offset - self.size = size - self.file_offset = file_offset - # caches for things that are "expensive" to compute - self._guid_str = None - self._guid_num = None - - @property - def guid_str(self): - if not self._guid_str: - self._guid_str = '-'.join('{:08x}'.format(g) for g in self.guid) - return self._guid_str - - @property - def guid_num(self): - if not self._guid_num: - self._guid_num = self.guid[3] + (self.guid[2] << 32) + (self.guid[1] << 64) + (self.guid[0] << 96) - return self._guid_num - - def __repr__(self): - guid_readable = '-'.join('{:08x}'.format(g) for g in self.guid) - return ''.format( - guid_readable, self.offset, self.size, self.file_offset) - - -class CustomFields: - def __init__(self): - self.size = 0 - self.version = 0 - self.count = 0 - - self._dict = dict() - - def __getitem__(self, item): - return self._dict.get(item, None) - - def __setitem__(self, key, value): - self._dict[key] = value - - def __str__(self): - return str(self._dict) - - def items(self): - return self._dict.items() - - def keys(self): - return self._dict.keys() - - def values(self): - return self._dict.values() - - @classmethod - def read(cls, bio): - _cf = cls() - - cf_start = bio.tell() - _cf.size = struct.unpack(' bool: - try: - logger.debug(f'Deleting "{path}", recursive={recursive}...') - if not recursive: - os.removedirs(path) - else: - shutil.rmtree(path) - except Exception as e: - logger.error(f'Failed deleting files with {e!r}') - return False - else: - return True - - -def delete_filelist(path: str, filenames: List[str], - delete_root_directory: bool = False, - silent: bool = False) -> bool: - dirs = set() - no_error = True - - # delete all files that were installed - for filename in filenames: - _dir, _fn = os.path.split(filename) - if _dir: - dirs.add(_dir) - - try: - os.remove(os.path.join(path, _dir, _fn)) - except Exception as e: - if not silent: - logger.error(f'Failed deleting file {filename} with {e!r}') - no_error = False - - # add intermediate directories that would have been missed otherwise - for _dir in sorted(dirs): - head, _ = os.path.split(_dir) - while head: - dirs.add(head) - head, _ = os.path.split(head) - - # remove all directories - for _dir in sorted(dirs, key=len, reverse=True): - try: - os.rmdir(os.path.join(path, _dir)) - except FileNotFoundError: - # directory has already been deleted, ignore that - continue - except Exception as e: - if not silent: - logger.error(f'Failed removing directory "{_dir}" with {e!r}') - no_error = False - - if delete_root_directory: - try: - os.rmdir(path) - except Exception as e: - if not silent: - logger.error(f'Removing game directory failed with {e!r}') - - return no_error - - -def validate_files(base_path: str, filelist: List[tuple], hash_type='sha1') -> Iterator[tuple]: - """ - Validates the files in filelist in path against the provided hashes - - :param base_path: path in which the files are located - :param filelist: list of tuples in format (path, hash [hex]) - :param hash_type: (optional) type of hash, default is sha1 - :return: list of files that failed hash check - """ - - if not filelist: - raise ValueError('No files to validate!') - - if not os.path.exists(base_path): - raise OSError('Path does not exist') - - for file_path, file_hash in filelist: - full_path = os.path.join(base_path, file_path) - # logger.debug(f'Checking "{file_path}"...') - - if not os.path.exists(full_path): - yield VerifyResult.FILE_MISSING, file_path, '' - continue - - try: - with open(full_path, 'rb') as f: - real_file_hash = hashlib.new(hash_type) - while chunk := f.read(1024 * 1024): - real_file_hash.update(chunk) - - result_hash = real_file_hash.hexdigest() - if file_hash != result_hash: - yield VerifyResult.HASH_MISMATCH, file_path, result_hash - else: - yield VerifyResult.HASH_MATCH, file_path, result_hash - except Exception as e: - logger.fatal(f'Could not verify "{file_path}"; opening failed with: {e!r}') - yield VerifyResult.OTHER_ERROR, file_path, '' - - -def clean_filename(filename): - return ''.join(i for i in filename if i not in '<>:"/\\|?*') diff --git a/custom_legendary/utils/manifests.py b/custom_legendary/utils/manifests.py deleted file mode 100644 index 744ffa28..00000000 --- a/custom_legendary/utils/manifests.py +++ /dev/null @@ -1,39 +0,0 @@ -from custom_legendary.models.manifest import Manifest - - -def combine_manifests(base_manifest: Manifest, delta_manifest: Manifest): - added = set() - # overwrite file elements with the ones from the delta manifest - for idx, file_elem in enumerate(base_manifest.file_manifest_list.elements): - try: - delta_file = delta_manifest.file_manifest_list.get_file_by_path(file_elem.filename) - base_manifest.file_manifest_list.elements[idx] = delta_file - added.add(delta_file.filename) - except ValueError: - pass - - # add other files that may be missing - for delta_file in delta_manifest.file_manifest_list.elements: - if delta_file.filename not in added: - base_manifest.file_manifest_list.elements.append(delta_file) - # update count and clear map - base_manifest.file_manifest_list.count = len(base_manifest.file_manifest_list.elements) - base_manifest.file_manifest_list._path_map = None - - # ensure guid map exists - try: - base_manifest.chunk_data_list.get_chunk_by_guid(0) - except: - pass - - # add new chunks from delta manifest to main manifest and again clear maps and update count - existing_chunk_guids = base_manifest.chunk_data_list._guid_int_map.keys() - - for chunk in delta_manifest.chunk_data_list.elements: - if chunk.guid_num not in existing_chunk_guids: - base_manifest.chunk_data_list.elements.append(chunk) - - base_manifest.chunk_data_list.count = len(base_manifest.chunk_data_list.elements) - base_manifest.chunk_data_list._guid_map = None - base_manifest.chunk_data_list._guid_int_map = None - base_manifest.chunk_data_list._path_map = None diff --git a/custom_legendary/utils/rolling_hash.py b/custom_legendary/utils/rolling_hash.py deleted file mode 100644 index 1903a99d..00000000 --- a/custom_legendary/utils/rolling_hash.py +++ /dev/null @@ -1,25 +0,0 @@ -# this is the rolling hash Epic uses, it appears to be a variation on CRC-64-ECMA - -hash_poly = 0xC96C5795D7870F42 -hash_table = [] - - -def _init(): - for i in range(256): - for _ in range(8): - if i & 1: - i >>= 1 - i ^= hash_poly - else: - i >>= 1 - hash_table.append(i) - - -def get_hash(data): - if not hash_table: - _init() - - h = 0 - for i in range(len(data)): - h = ((h << 1 | h >> 63) ^ hash_table[data[i]]) & 0xffffffffffffffff - return h diff --git a/custom_legendary/utils/savegame_helper.py b/custom_legendary/utils/savegame_helper.py deleted file mode 100644 index baaa1cfc..00000000 --- a/custom_legendary/utils/savegame_helper.py +++ /dev/null @@ -1,165 +0,0 @@ -import logging -import os -from datetime import datetime -from fnmatch import fnmatch -from hashlib import sha1 -from io import BytesIO -from tempfile import TemporaryFile - -from custom_legendary.models.chunk import Chunk -from custom_legendary.models.manifest import \ - Manifest, ManifestMeta, CDL, FML, CustomFields, FileManifest, ChunkPart, ChunkInfo - - -def _filename_matches(filename, patterns): - """ - Helper to determine if a filename matches the filter patterns - - :param filename: name of the file - :param patterns: list of patterns to match against - :return: - """ - - for pattern in patterns: - if pattern.endswith('/'): - # pat is a directory, check if path starts with it - if filename.startswith(pattern): - return True - elif fnmatch(filename, pattern): - return True - - return False - - -class SaveGameHelper: - def __init__(self): - self.files = dict() - self.log = logging.getLogger('SGH') - - def finalize_chunk(self, chunk: Chunk): - ci = ChunkInfo() - ci.guid = chunk.guid - ci.hash = chunk.hash - ci.sha_hash = chunk.sha_hash - # use a temporary file for uploading - _tmp_file = TemporaryFile() - self.files[ci.path] = _tmp_file - # write() returns file size and also sets the uncompressed size - ci.file_size = chunk.write(_tmp_file) - ci.window_size = chunk.uncompressed_size - _tmp_file.seek(0) - return ci - - def package_savegame(self, input_folder: str, app_name: str = '', - epic_id: str = '', cloud_folder: str = '', - include_filter: list = None, - exclude_filter: list = None, - manifest_dt: datetime = None): - """ - :param input_folder: Folder to be packaged into chunks/manifest - :param app_name: App name for savegame being stored - :param epic_id: Epic account ID - :param cloud_folder: Folder the savegame resides in (based on game metadata) - :param include_filter: list of patterns for files to include (excludes all others) - :param exclude_filter: list of patterns for files to exclude (includes all others) - :param manifest_dt: datetime for the manifest name (optional) - :return: - """ - m = Manifest() - m.meta = ManifestMeta() - m.chunk_data_list = CDL() - m.file_manifest_list = FML() - m.custom_fields = CustomFields() - # create metadata for savegame - m.meta.app_name = f'{app_name}{epic_id}' - if not manifest_dt: - manifest_dt = datetime.utcnow() - m.meta.build_version = manifest_dt.strftime('%Y.%m.%d-%H.%M.%S') - m.custom_fields['CloudSaveFolder'] = cloud_folder - - self.log.info(f'Packing savegame for "{app_name}", input folder: {input_folder}') - files = [] - for _dir, _, _files in os.walk(input_folder): - for _file in _files: - _file_path = os.path.join(_dir, _file) - _file_path_rel = os.path.relpath(_file_path, input_folder).replace('\\', '/') - - if include_filter and not _filename_matches(_file_path_rel, include_filter): - self.log.debug(f'Excluding "{_file_path_rel}" (does not match include filter)') - continue - elif exclude_filter and _filename_matches(_file_path_rel, exclude_filter): - self.log.debug(f'Excluding "{_file_path_rel}" (does match exclude filter)') - continue - - files.append(_file_path) - - if not files: - if exclude_filter or include_filter: - self.log.warning('No save files matching the specified filters have been found.') - return self.files - - chunk_num = 0 - cur_chunk = None - cur_buffer = None - - for _file in sorted(files, key=str.casefold): - s = os.stat(_file) - f = FileManifest() - # get relative path for manifest - f.filename = os.path.relpath(_file, input_folder).replace('\\', '/') - self.log.debug(f'Processing file "{f.filename}"') - f.file_size = s.st_size - fhash = sha1() - - with open(_file, 'rb') as cf: - while remaining := s.st_size - cf.tell(): - if not cur_chunk: # create new chunk - cur_chunk = Chunk() - if cur_buffer: - cur_buffer.close() - cur_buffer = BytesIO() - chunk_num += 1 - - # create chunk part and write it to chunk buffer - cp = ChunkPart(guid=cur_chunk.guid, offset=cur_buffer.tell(), - size=min(remaining, 1024 * 1024 - cur_buffer.tell()), - file_offset=cf.tell()) - _tmp = cf.read(cp.size) - if not _tmp: - self.log.warning(f'Got EOF for "{f.filename}" with {remaining} bytes remaining! ' - f'File may have been corrupted/modified.') - break - - cur_buffer.write(_tmp) - fhash.update(_tmp) # update sha1 hash with new data - f.chunk_parts.append(cp) - - if cur_buffer.tell() >= 1024 * 1024: - cur_chunk.data = cur_buffer.getvalue() - ci = self.finalize_chunk(cur_chunk) - self.log.info(f'Chunk #{chunk_num} "{ci.path}" created') - # add chunk to CDL - m.chunk_data_list.elements.append(ci) - cur_chunk = None - - f.hash = fhash.digest() - m.file_manifest_list.elements.append(f) - - # write remaining chunk if it exists - if cur_chunk: - cur_chunk.data = cur_buffer.getvalue() - ci = self.finalize_chunk(cur_chunk) - self.log.info(f'Chunk #{chunk_num} "{ci.path}" created') - m.chunk_data_list.elements.append(ci) - cur_buffer.close() - - # Finally write/serialize manifest into another temporary file - _m_filename = f'manifests/{m.meta.build_version}.manifest' - _tmp_file = TemporaryFile() - _m_size = m.write(_tmp_file) - _tmp_file.seek(0) - self.log.info(f'Manifest "{_m_filename}" written ({_m_size} bytes)') - self.files[_m_filename] = _tmp_file - - # return dict with created files for uploading/whatever - return self.files diff --git a/custom_legendary/utils/selective_dl.py b/custom_legendary/utils/selective_dl.py deleted file mode 100644 index c0ec31a2..00000000 --- a/custom_legendary/utils/selective_dl.py +++ /dev/null @@ -1,38 +0,0 @@ -# This file contains definitions for selective downloading for supported games -# coding: utf-8 - -_cyberpunk_sdl = { - 'de': {'tags': ['voice_de_de'], 'name': 'Deutsch'}, - 'es': {'tags': ['voice_es_es'], 'name': 'español (España)'}, - 'fr': {'tags': ['voice_fr_fr'], 'name': 'français'}, - 'it': {'tags': ['voice_it_it'], 'name': 'italiano'}, - 'ja': {'tags': ['voice_ja_jp'], 'name': '日本語'}, - 'ko': {'tags': ['voice_ko_kr'], 'name': '한국어'}, - 'pl': {'tags': ['voice_pl_pl'], 'name': 'polski'}, - 'pt': {'tags': ['voice_pt_br'], 'name': 'português brasileiro'}, - 'ru': {'tags': ['voice_ru_ru'], 'name': 'русский'}, - 'cn': {'tags': ['voice_zh_cn'], 'name': '中文（中国）'} -} - -_fortnite_sdl = { - '__required': {'tags': ['chunk0', 'chunk10'], 'name': 'Fortnite Core'}, - 'stw': {'tags': ['chunk11', 'chunk11optional'], 'name': 'Fortnite Save the World'}, - 'hd_textures': {'tags': ['chunk10optional'], 'name': 'High Resolution Textures'}, - 'lang_de': {'tags': ['chunk2'], 'name': '(Language Pack) Deutsch'}, - 'lang_fr': {'tags': ['chunk5'], 'name': '(Language Pack) français'}, - 'lang_pl': {'tags': ['chunk7'], 'name': '(Language Pack) polski'}, - 'lang_ru': {'tags': ['chunk8'], 'name': '(Language Pack) русский'}, - 'lang_cn': {'tags': ['chunk9'], 'name': '(Language Pack) 中文（中国）'} -} - -games = { - 'Fortnite': _fortnite_sdl, - 'Ginger': _cyberpunk_sdl -} - - -def get_sdl_appname(app_name): - for k in games.keys(): - if app_name.startswith(k): - return k - return None diff --git a/custom_legendary/utils/wine_helpers.py b/custom_legendary/utils/wine_helpers.py deleted file mode 100644 index 962d4663..00000000 --- a/custom_legendary/utils/wine_helpers.py +++ /dev/null @@ -1,17 +0,0 @@ -import configparser -import os - - -def read_registry(wine_pfx): - reg = configparser.ConfigParser(comment_prefixes=(';', '#', '/', 'WINE'), allow_no_value=True) - reg.optionxform = str - reg.read(os.path.join(wine_pfx, 'user.reg')) - return reg - - -def get_shell_folders(registry, wine_pfx): - folders = dict() - for k, v in registry['Software\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\Shell Folders'].items(): - path_cleaned = v.strip('"').strip().replace('\\\\', '/').replace('C:/', '') - folders[k.strip('"').strip()] = os.path.join(wine_pfx, 'drive_c', path_cleaned) - return folders diff --git a/legendary b/legendary new file mode 160000 index 00000000..a561cd8f --- /dev/null +++ b/legendary @@ -0,0 +1 @@ +Subproject commit a561cd8f0d7316867e3b3d20dfdb47c8208f63f4 diff --git a/rare/__main__.py b/rare/__main__.py index ddfb8557..5bdb8987 100644 --- a/rare/__main__.py +++ b/rare/__main__.py @@ -1,14 +1,15 @@ #!/usr/bin/python import os +import pathlib +import sys from argparse import ArgumentParser from rare import __version__, data_dir -from rare.utils import singleton, utils +from rare.utils import singleton def main(): - # CLI Options parser = ArgumentParser() @@ -30,9 +31,11 @@ def main(): args = parser.parse_args() if args.desktop_shortcut: + from rare.utils import utils utils.create_rare_desktop_link("desktop") print("Link created") if args.startmenu_shortcut: + from rare.utils import utils utils.create_rare_desktop_link("start_menu") print("link created") @@ -67,5 +70,5 @@ if __name__ == '__main__': import multiprocessing multiprocessing.freeze_support() - + sys.path.insert(0, os.path.join(pathlib.Path(__file__).parent.parent.absolute(), "legendary")) main()