[cli/core] Fix fetching more than 1000 entitlements

This should fix things like the fake Fortnite "games" (Lego Fortnite etc.)

.github/workflows/python.yml

@@ -27,6 +27,7 @@ jobs:
         setuptools
         pyinstaller
         requests
+        filelock
 
     - name: Optional dependencies (WebView)
       run: pip3 install --upgrade pywebview

legendary/__init__.py

@@ -1,4 +1,4 @@
 """Legendary!"""
 
-__version__ = '0.20.33'
-__codename__ = 'Undue Alarm'
+__version__ = '0.20.34'
+__codename__ = 'Direct Intervention'

legendary/api/egs.py

@@ -186,13 +186,24 @@ class EPCAPI:
         r.raise_for_status()
         return r.json()
 
-    def get_user_entitlements(self):
+    def get_user_entitlements(self, start=0):
         user_id = self.user.get('account_id')
         r = self.session.get(f'https://{self._entitlements_host}/entitlement/api/account/{user_id}/entitlements',
-                             params=dict(start=0, count=5000), timeout=self.request_timeout)
+                             params=dict(start=start, count=1000), timeout=self.request_timeout)
         r.raise_for_status()
         return r.json()
 
+    def get_user_entitlements_full(self):
+        ret = []
+
+        while True:
+            resp = self.get_user_entitlements(start=len(ret))
+            ret.extend(resp)
+            if len(resp) < 1000:
+                break
+
+        return ret
+
     def get_game_info(self, namespace, catalog_item_id, timeout=None):
         r = self.session.get(f'https://{self._catalog_host}/catalog/api/shared/namespace/{namespace}/bulk/items',
                              params=dict(id=catalog_item_id, includeDLCDetails=True, includeMainGameDetails=True,

legendary/cli.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # coding: utf-8
 
 import argparse
@@ -104,11 +104,11 @@ class LegendaryCLI:
 
                 if not egl_wine_pfx:
                     logger.info('Please enter the path to the Wine prefix that has EGL installed')
-                    wine_pfx = input('Path [empty input to quit]: ').strip()
-                    if not wine_pfx:
+                    egl_wine_pfx = input('Path [empty input to quit]: ').strip()
+                    if not egl_wine_pfx:
                         print('Empty input, quitting...')
                         exit(0)
-                    if not os.path.exists(wine_pfx) and os.path.isdir(wine_pfx):
+                    if not os.path.exists(egl_wine_pfx) and os.path.isdir(egl_wine_pfx):
                         print('Path is invalid (does not exist)!')
                         exit(1)
 
@@ -144,7 +144,7 @@ class LegendaryCLI:
 
         exchange_token = ''
         auth_code = ''
-        if not args.auth_code and not args.session_id:
+        if not args.auth_code and not args.session_id and not args.ex_token:
             # only import here since pywebview import is slow
             from legendary.utils.webview_login import webview_available, do_webview_login
 
@@ -162,7 +162,8 @@ class LegendaryCLI:
                 else:
                     auth_code = auth_code.strip('"')
             else:
-                if do_webview_login(callback_code=self.core.auth_ex_token):
+                if do_webview_login(callback_code=self.core.auth_ex_token,
+                                    user_agent=f'EpicGamesLauncher/{self.core.get_egl_version()}'):
                     logger.info(f'Successfully logged in as "{self.core.lgd.userdata["displayName"]}" via WebView')
                 else:
                     logger.error('WebView login attempt failed, please see log for details.')
@@ -371,6 +372,8 @@ class LegendaryCLI:
 
         if args.install_tag:
             files = [fm for fm in files if args.install_tag in fm.install_tags]
+        elif args.install_tag is not None:
+            files = [fm for fm in files if not fm.install_tags]
 
         if args.hashlist:
             for fm in files:
@@ -565,6 +568,7 @@ class LegendaryCLI:
 
     def launch_game(self, args, extra):
         app_name = self._resolve_aliases(args.app_name)
+        addon_app_name = None
 
         # Interactive CrossOver setup
         if args.crossover and sys_platform == 'darwin':
@@ -575,12 +579,19 @@ class LegendaryCLI:
             return self._launch_origin(args)
 
         igame = self.core.get_installed_game(app_name)
+        if (not igame or not igame.executable) and (game := self.core.get_game(app_name)) is not None:
+            # override installed game with base title
+            if game.is_launchable_addon:
+                addon_app_name = app_name
+                app_name = game.metadata['mainGameItem']['releaseInfo'][0]['appId']
+                igame = self.core.get_installed_game(app_name)
+
         if not igame:
             logger.error(f'Game {app_name} is not currently installed!')
             exit(1)
 
-        if igame.is_dlc:
-            logger.error(f'{app_name} is DLC; please launch the base game instead!')
+        if igame.is_dlc and not igame.executable:
+            logger.error(f'{app_name} is DLC without an executable; please launch the base game instead!')
             exit(1)
 
         if not os.path.exists(igame.install_path):
@@ -619,7 +630,8 @@ class LegendaryCLI:
                                                  disable_wine=args.no_wine,
                                                  executable_override=args.executable_override,
                                                  crossover_app=args.crossover_app,
-                                                 crossover_bottle=args.crossover_bottle)
+                                                 crossover_bottle=args.crossover_bottle,
+                                                 addon_app_name=addon_app_name)
 
         if args.set_defaults:
             self.core.lgd.config[app_name] = dict()
@@ -779,6 +791,8 @@ class LegendaryCLI:
                          f'wrapper in the configuration file or command line. See the README for details.')
             return
 
+        # You cannot launch a URI without start.exe
+        command.append('start')
         command.append(origin_uri)
         if args.dry_run:
             if cmd:
@@ -909,7 +923,7 @@ class LegendaryCLI:
             if config_tags:
                 self.core.lgd.config.remove_option(game.app_name, 'install_tags')
                 config_tags = None
-            self.core.lgd.config.set(game.app_name, 'disable_sdl', True)
+            self.core.lgd.config.set(game.app_name, 'disable_sdl', 'true')
             sdl_enabled = False
         # just disable SDL, but keep config tags that have been manually specified
         elif config_disable_sdl or args.disable_sdl:
@@ -963,7 +977,8 @@ class LegendaryCLI:
                                                           disable_delta=args.disable_delta,
                                                           override_delta_manifest=args.override_delta_manifest,
                                                           preferred_cdn=args.preferred_cdn,
-                                                          disable_https=args.disable_https)
+                                                          disable_https=args.disable_https,
+                                                          bind_ip=args.bind_ip)
 
         # game is either up-to-date or hasn't changed, so we have nothing to do
         if not analysis.dl_size:
@@ -984,6 +999,10 @@ class LegendaryCLI:
                 self.core.uninstall_tag(old_igame)
                 self.core.install_game(old_igame)
 
+            if old_igame.install_tags:
+                self.core.lgd.config.set(game.app_name, 'install_tags', ','.join(old_igame.install_tags))
+                self.core.lgd.save_config()
+
             # check if the version changed, this can happen for DLC that gets a version bump with no actual file changes
             if old_igame and old_igame.version != igame.version:
                 old_igame.version = igame.version
@@ -1225,7 +1244,7 @@ class LegendaryCLI:
                        key=lambda a: a.filename.lower())
 
         # build list of hashes
-        if config_tags := self.core.lgd.config.get(args.app_name, 'install_tags', fallback=None):
+        if (config_tags := self.core.lgd.config.get(args.app_name, 'install_tags', fallback=None)) is not None:
             install_tags = set(i.strip() for i in config_tags.split(','))
             file_list = [
                 (f.filename, f.sha_hash.hex())
@@ -1638,7 +1657,7 @@ class LegendaryCLI:
             else:
                 logger.info('Game not installed and offline mode enabled, cannot load manifest.')
         elif game:
-            entitlements = self.core.egs.get_user_entitlements()
+            entitlements = self.core.egs.get_user_entitlements_full()
             egl_meta = self.core.egs.get_game_info(game.namespace, game.catalog_item_id)
             game.metadata = egl_meta
             # Get manifest if asset exists for current platform
@@ -1676,7 +1695,7 @@ class LegendaryCLI:
             # Find custom launch options, if available
             launch_options = []
             i = 1
-            while f'extraLaunchOption_{i:03d}_Name' in game.metadata['customAttributes']:
+            while f'extraLaunchOption_{i:03d}_Name' in game.metadata.get('customAttributes', {}):
                 launch_options.append((
                     game.metadata['customAttributes'][f'extraLaunchOption_{i:03d}_Name']['value'],
                     game.metadata['customAttributes'][f'extraLaunchOption_{i:03d}_Args']['value']
@@ -1694,6 +1713,9 @@ class LegendaryCLI:
             else:
                 game_infos.append(InfoItem('Extra launch options', 'launch_options', None, []))
 
+            game_infos.append(InfoItem('Command Line', 'command_line', game.additional_command_line,
+                                       game.additional_command_line))
+
             # list all owned DLC based on entitlements
             if entitlements and not game.is_dlc:
                 owned_entitlements = {i['entitlementName'] for i in entitlements}
@@ -2024,7 +2046,7 @@ class LegendaryCLI:
             redeemed = {k['gameId'] for k in key_list if k['redeemedOnUplay']}
 
             games = self.core.get_game_list()
-            entitlements = self.core.egs.get_user_entitlements()
+            entitlements = self.core.egs.get_user_entitlements_full()
             owned_entitlements = {i['entitlementName'] for i in entitlements}
 
             uplay_games = []
@@ -2774,6 +2796,8 @@ def main():
                                 help='Automatically install all DLCs with the base game')
     install_parser.add_argument('--skip-dlcs', dest='skip_dlcs', action='store_true',
                                 help='Do not ask about installing DLCs.')
+    install_parser.add_argument('--bind', dest='bind_ip', action='store', metavar='<IPs>', type=str,
+                                help='Comma-separated list of IPs to bind to for downloading')
 
     uninstall_parser.add_argument('--keep-files', dest='keep_files', action='store_true',
                                   help='Keep files but remove game from Legendary database')

legendary/core.py

@@ -239,7 +239,7 @@ class LegendaryCore:
             userdata = self.egs.start_session(lock.data['refresh_token'])
         except InvalidCredentialsError:
             self.log.error('Stored credentials are no longer valid! Please login again.')
-            self.lgd.invalidate_userdata()
+            lock.clear()
             return False
         except (HTTPError, ConnectionError) as e:
             self.log.error(f'HTTP request for login failed: {e!r}, please try again later.')
@@ -301,6 +301,9 @@ class LegendaryCore:
         if lgd_config := version_info.get('legendary_config'):
             self.webview_killswitch = lgd_config.get('webview_killswitch', False)
 
+    def get_egl_version(self):
+        return self._egl_version
+
     def get_update_info(self):
         return self.lgd.get_cached_version()['data'].get('release_info')
 
@@ -517,12 +520,16 @@ class LegendaryCore:
         _dlc = defaultdict(list)
         # get all the appnames we have to ignore
         ignore = set(i.app_name for i in self.get_assets())
+        # broken old app name that we should always ignore
+        ignore |= {'1'}
 
         for libitem in self.egs.get_library_items():
             if libitem['namespace'] == 'ue' and skip_ue:
                 continue
             if libitem['appName'] in ignore:
                 continue
+            if libitem['sandboxType'] == 'PRIVATE':
+                continue
 
             game = self.lgd.get_game_meta(libitem['appName'])
             if not game or force_refresh:
@@ -682,9 +689,10 @@ class LegendaryCore:
                               disable_wine: bool = False,
                               executable_override: str = None,
                               crossover_app: str = None,
-                              crossover_bottle: str = None) -> LaunchParameters:
+                              crossover_bottle: str = None,
+                              addon_app_name: str = None) -> LaunchParameters:
         install = self.lgd.get_installed_game(app_name)
-        game = self.lgd.get_game_meta(app_name)
+        game = self.lgd.get_game_meta(addon_app_name if addon_app_name else app_name)
 
         # Disable wine for non-Windows executables (e.g. native macOS)
         if not install.platform.startswith('Win'):
@@ -726,6 +734,13 @@ class LegendaryCore:
                 self.log.warning(f'Parsing predefined launch parameters failed with: {e!r}, '
                                  f'input: {install.launch_parameters}')
 
+        if meta_args := game.additional_command_line:
+            try:
+                params.game_parameters.extend(shlex.split(meta_args.strip(), posix=False))
+            except ValueError as e:
+                self.log.warning(f'Parsing metadata launch parameters failed with: {e!r}, '
+                                 f'input: {install.launch_parameters}')
+
         game_token = ''
         if not offline:
             self.log.info('Getting authentication token...')
@@ -1292,7 +1307,7 @@ class LegendaryCore:
                          repair: bool = False, repair_use_latest: bool = False,
                          disable_delta: bool = False, override_delta_manifest: str = '',
                          egl_guid: str = '', preferred_cdn: str = None,
-                         disable_https: bool = False) -> (DLManager, AnalysisResult, ManifestMeta):
+                         disable_https: bool = False, bind_ip: str = None) -> (DLManager, AnalysisResult, ManifestMeta):
         # load old manifest
         old_manifest = None
 
@@ -1459,7 +1474,7 @@ class LegendaryCore:
 
         dlm = DLManager(install_path, base_url, resume_file=resume_file, status_q=status_q,
                         max_shared_memory=max_shm * 1024 * 1024, max_workers=max_workers,
-                        dl_timeout=dl_timeout)
+                        dl_timeout=dl_timeout, bind_ip=bind_ip)
         anlres = dlm.run_analysis(manifest=new_manifest, old_manifest=old_manifest,
                                   patch=not disable_patching, resume=not force,
                                   file_prefix_filter=file_prefix_filter,

legendary/downloader/mp/manager.py

@@ -22,7 +22,7 @@ from legendary.models.manifest import ManifestComparison, Manifest
 class DLManager(Process):
     def __init__(self, download_dir, base_url, cache_dir=None, status_q=None,
                  max_workers=0, update_interval=1.0, dl_timeout=10, resume_file=None,
-                 max_shared_memory=1024 * 1024 * 1024):
+                 max_shared_memory=1024 * 1024 * 1024, bind_ip=None):
         super().__init__(name='DLManager')
         self.log = logging.getLogger('DLM')
         self.proc_debug = False
@@ -37,8 +37,11 @@ class DLManager(Process):
         self.writer_queue = None
         self.dl_result_q = None
         self.writer_result_q = None
+
+        # Worker stuff
         self.max_workers = max_workers or min(cpu_count() * 2, 16)
         self.dl_timeout = dl_timeout
+        self.bind_ips = [] if not bind_ip else bind_ip.split(',')
 
         # Analysis stuff
         self.analysis = None
@@ -655,10 +658,15 @@ class DLManager(Process):
         self.writer_result_q = MPQueue(-1)
 
         self.log.info(f'Starting download workers...')
+
+        bind_ip = None
         for i in range(self.max_workers):
+            if self.bind_ips:
+                bind_ip = self.bind_ips[i % len(self.bind_ips)]
+
             w = DLWorker(f'DLWorker {i + 1}', self.dl_worker_queue, self.dl_result_q,
                          self.shared_memory.name, logging_queue=self.logging_queue,
-                         dl_timeout=self.dl_timeout)
+                         dl_timeout=self.dl_timeout, bind_addr=bind_ip)
             self.children.append(w)
             w.start()
 

legendary/downloader/mp/workers.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 
 import os
-import requests
 import time
 import logging
 
@@ -10,6 +9,9 @@ from multiprocessing import Process
 from multiprocessing.shared_memory import SharedMemory
 from queue import Empty
 
+import requests
+from requests.adapters import HTTPAdapter, DEFAULT_POOLBLOCK
+
 from legendary.models.chunk import Chunk
 from legendary.models.downloading import (
     DownloaderTask, DownloaderTaskResult,
@@ -18,9 +20,22 @@ from legendary.models.downloading import (
 )
 
 
+class BindingHTTPAdapter(HTTPAdapter):
+    def __init__(self, addr):
+        self.__attrs__.append('addr')
+        self.addr = addr
+        super().__init__()
+
+    def init_poolmanager(
+            self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
+    ):
+        pool_kwargs['source_address'] = (self.addr, 0)
+        super().init_poolmanager(connections, maxsize, block, **pool_kwargs)
+
+
 class DLWorker(Process):
     def __init__(self, name, queue, out_queue, shm, max_retries=7,
-                 logging_queue=None, dl_timeout=10):
+                 logging_queue=None, dl_timeout=10, bind_addr=None):
         super().__init__(name=name)
         self.q = queue
         self.o_q = out_queue
@@ -34,6 +49,12 @@ class DLWorker(Process):
         self.logging_queue = logging_queue
         self.dl_timeout = float(dl_timeout) if dl_timeout else 10.0
 
+        # optionally bind an address
+        if bind_addr:
+            adapter = BindingHTTPAdapter(bind_addr)
+            self.session.mount('https://', adapter)
+            self.session.mount('http://', adapter)
+
     def run(self):
         # we have to fix up the logger before we can start
         _root = logging.getLogger()
@@ -105,11 +126,12 @@ class DLWorker(Process):
 
             # decompress stuff
             try:
-                size = len(chunk.data)
+                data = chunk.data
+                size = len(data)
                 if size > job.shm.size:
                     logger.fatal('Downloaded chunk is longer than SharedMemorySegment!')
 
-                self.shm.buf[job.shm.offset:job.shm.offset + size] = bytes(chunk.data)
+                self.shm.buf[job.shm.offset:job.shm.offset + size] = data
                 del chunk
                 self.o_q.put(DownloaderTaskResult(success=True, size_decompressed=size,
                                                   size_downloaded=compressed, **job.__dict__))
@@ -250,7 +272,7 @@ class FileWorker(Process):
                     if j.shared_memory:
                         shm_offset = j.shared_memory.offset + j.chunk_offset
                         shm_end = shm_offset + j.chunk_size
-                        current_file.write(self.shm.buf[shm_offset:shm_end].tobytes())
+                        current_file.write(self.shm.buf[shm_offset:shm_end])
                     elif j.cache_file:
                         with open(os.path.join(self.cache_path, j.cache_file), 'rb') as f:
                             if j.chunk_offset:

legendary/lfs/lgndry.py

@@ -26,7 +26,9 @@ class LGDLFS:
     def __init__(self, config_file=None):
         self.log = logging.getLogger('LGDLFS')
 
-        if config_path := os.environ.get('XDG_CONFIG_HOME'):
+        if config_path := os.environ.get('LEGENDARY_CONFIG_PATH'):
+            self.path = config_path
+        elif config_path := os.environ.get('XDG_CONFIG_HOME'):
             self.path = os.path.join(config_path, 'legendary')
         else:
             self.path = os.path.expanduser('~/.config/legendary')

legendary/models/game.py

@@ -91,6 +91,18 @@ class Game:
     def supports_mac_cloud_saves(self):
         return self.metadata and (self.metadata.get('customAttributes', {}).get('CloudSaveFolder_MAC') is not None)
 
+    @property
+    def additional_command_line(self):
+        if not self.metadata:
+            return None
+        return self.metadata.get('customAttributes', {}).get('AdditionalCommandLine', {}).get('value', None)
+
+    @property
+    def is_launchable_addon(self):
+        if not self.metadata:
+            return False
+        return any(m['path'] == 'addons/launchable' for m in self.metadata.get('categories', []))
+
     @property
     def catalog_item_id(self):
         if not self.metadata:

legendary/utils/webview_login.py

@@ -124,7 +124,7 @@ class MockLauncher:
             self.window.load_url(logout_url)
 
 
-def do_webview_login(callback_sid=None, callback_code=None):
+def do_webview_login(callback_sid=None, callback_code=None, user_agent=None):
     api = MockLauncher(callback_sid=callback_sid, callback_code=callback_code)
     url = login_url
 
@@ -143,7 +143,7 @@ def do_webview_login(callback_sid=None, callback_code=None):
     window.events.loaded += api.on_loaded
 
     try:
-        webview.start()
+        webview.start(user_agent=user_agent)
     except Exception as we:
         logger.error(f'Running webview failed with {we!r}. If this error persists try the manual '
                      f'login process by adding --disable-webview to your command line.')