Rename --exclude-id(-file) to --skip-id(-file)
This commit is contained in:
parent
f768a7d61c
commit
200916a150
|
@ -131,10 +131,10 @@ The following options are common between both the `archive` and `download` comma
|
||||||
|
|
||||||
The following options apply only to the `download` command. This command downloads the files and resources linked to in the submission, or a text submission itself, to the disk in the specified directory.
|
The following options apply only to the `download` command. This command downloads the files and resources linked to in the submission, or a text submission itself, to the disk in the specified directory.
|
||||||
|
|
||||||
- `--exclude-id`
|
- `--skip-id`
|
||||||
- This will skip the download of any submission with the ID provided
|
- This will skip the download of any submission with the ID provided
|
||||||
- Can be specified multiple times
|
- Can be specified multiple times
|
||||||
- `--exclude-id-file`
|
- `--skip-id-file`
|
||||||
- This will skip the download of any submission with any of the IDs in the files provided
|
- This will skip the download of any submission with any of the IDs in the files provided
|
||||||
- Can be specified multiple times
|
- Can be specified multiple times
|
||||||
- Format is one ID per line
|
- Format is one ID per line
|
||||||
|
|
|
@ -45,14 +45,14 @@ def cli():
|
||||||
|
|
||||||
|
|
||||||
@cli.command('download')
|
@cli.command('download')
|
||||||
@click.option('--exclude-id', default=None, multiple=True)
|
|
||||||
@click.option('--exclude-id-file', default=None, multiple=True)
|
|
||||||
@click.option('--file-scheme', default=None, type=str)
|
@click.option('--file-scheme', default=None, type=str)
|
||||||
@click.option('--folder-scheme', default=None, type=str)
|
@click.option('--folder-scheme', default=None, type=str)
|
||||||
@click.option('--make-hard-links', is_flag=True, default=None)
|
@click.option('--make-hard-links', is_flag=True, default=None)
|
||||||
@click.option('--max-wait-time', type=int, default=None)
|
@click.option('--max-wait-time', type=int, default=None)
|
||||||
@click.option('--no-dupes', is_flag=True, default=None)
|
@click.option('--no-dupes', is_flag=True, default=None)
|
||||||
@click.option('--search-existing', is_flag=True, default=None)
|
@click.option('--search-existing', is_flag=True, default=None)
|
||||||
|
@click.option('--skip-id', default=None, multiple=True)
|
||||||
|
@click.option('--skip-id-file', default=None, multiple=True)
|
||||||
@click.option('--skip-format', default=None, multiple=True)
|
@click.option('--skip-format', default=None, multiple=True)
|
||||||
@click.option('--skip-domain', default=None, multiple=True)
|
@click.option('--skip-domain', default=None, multiple=True)
|
||||||
@click.option('--skip-subreddit', default=None, multiple=True)
|
@click.option('--skip-subreddit', default=None, multiple=True)
|
||||||
|
|
|
@ -13,8 +13,8 @@ class Configuration(Namespace):
|
||||||
self.authenticate = False
|
self.authenticate = False
|
||||||
self.config = None
|
self.config = None
|
||||||
self.directory: str = '.'
|
self.directory: str = '.'
|
||||||
self.exclude_id = []
|
self.skip_id = []
|
||||||
self.exclude_id_file = []
|
self.skip_id_file = []
|
||||||
self.limit: Optional[int] = None
|
self.limit: Optional[int] = None
|
||||||
self.link: list[str] = []
|
self.link: list[str] = []
|
||||||
self.log: Optional[str] = None
|
self.log: Optional[str] = None
|
||||||
|
|
|
@ -460,8 +460,8 @@ class RedditDownloader:
|
||||||
|
|
||||||
def _read_excluded_ids(self) -> set[str]:
|
def _read_excluded_ids(self) -> set[str]:
|
||||||
out = []
|
out = []
|
||||||
out.extend(self.args.exclude_id)
|
out.extend(self.args.skip_id)
|
||||||
for id_file in self.args.exclude_id_file:
|
for id_file in self.args.skip_id_file:
|
||||||
id_file = Path(id_file).resolve().expanduser()
|
id_file = Path(id_file).resolve().expanduser()
|
||||||
if not id_file.exists():
|
if not id_file.exists():
|
||||||
logger.warning(f'ID exclusion file at {id_file} does not exist')
|
logger.warning(f'ID exclusion file at {id_file} does not exist')
|
||||||
|
|
|
@ -9,7 +9,7 @@ Due to the verboseness of the logs, a great deal of information can be gathered
|
||||||
|
|
||||||
## Extract all Successfully Downloaded IDs
|
## Extract all Successfully Downloaded IDs
|
||||||
|
|
||||||
This script is contained [here](extract_successful_ids.sh) and will result in a file that contains the IDs of everything that was successfully downloaded without an error. That is, a list will be created of submissions that, with the `--exclude-id-file` option, can be used so that the BDFR will not attempt to redownload these submissions/comments. This is likely to cause a performance increase, especially when the BDFR run finds many resources.
|
This script is contained [here](extract_successful_ids.sh) and will result in a file that contains the IDs of everything that was successfully downloaded without an error. That is, a list will be created of submissions that, with the `--skip-id-file` option, can be used so that the BDFR will not attempt to redownload these submissions/comments. This is likely to cause a performance increase, especially when the BDFR run finds many resources.
|
||||||
|
|
||||||
The script can be used with the following signature:
|
The script can be used with the following signature:
|
||||||
|
|
||||||
|
|
|
@ -456,7 +456,7 @@ def test_excluded_ids(test_ids: tuple[str], test_excluded: tuple[str], expected_
|
||||||
def test_read_excluded_submission_ids_from_file(downloader_mock: MagicMock, tmp_path: Path):
|
def test_read_excluded_submission_ids_from_file(downloader_mock: MagicMock, tmp_path: Path):
|
||||||
test_file = tmp_path / 'test.txt'
|
test_file = tmp_path / 'test.txt'
|
||||||
test_file.write_text('aaaaaa\nbbbbbb')
|
test_file.write_text('aaaaaa\nbbbbbb')
|
||||||
downloader_mock.args.exclude_id_file = [test_file]
|
downloader_mock.args.skip_id_file = [test_file]
|
||||||
results = RedditDownloader._read_excluded_ids(downloader_mock)
|
results = RedditDownloader._read_excluded_ids(downloader_mock)
|
||||||
assert results == {'aaaaaa', 'bbbbbb'}
|
assert results == {'aaaaaa', 'bbbbbb'}
|
||||||
|
|
||||||
|
|
|
@ -299,7 +299,7 @@ def test_cli_download_use_default_config(tmp_path: Path):
|
||||||
@pytest.mark.reddit
|
@pytest.mark.reddit
|
||||||
@pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests')
|
@pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests')
|
||||||
@pytest.mark.parametrize('test_args', (
|
@pytest.mark.parametrize('test_args', (
|
||||||
['-l', 'm2601g', '--exclude-id', 'm2601g'],
|
['-l', 'm2601g', '--skip-id', 'm2601g'],
|
||||||
))
|
))
|
||||||
def test_cli_download_links_exclusion(test_args: list[str], tmp_path: Path):
|
def test_cli_download_links_exclusion(test_args: list[str], tmp_path: Path):
|
||||||
runner = CliRunner()
|
runner = CliRunner()
|
||||||
|
|
Loading…
Reference in a new issue