Merge pull request #622 from stared/yaml-options
This commit is contained in:
commit
7ae318fb20
34
README.md
34
README.md
|
@ -68,6 +68,31 @@ python3 -m bdfr download ./path/to/output --subreddit 'Python, all, mindustry' -
|
||||||
python3 -m bdfr archive ./path/to/output --subreddit all --format yaml -L 500 --folder-scheme ''
|
python3 -m bdfr archive ./path/to/output --subreddit all --format yaml -L 500 --folder-scheme ''
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Alternatively, you can pass options through a YAML file.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 -m bdfr download ./path/to/output --opts my_opts.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
For example, running it with the following file
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
skip: [mp4, avi]
|
||||||
|
file_scheme: "{UPVOTES}_{REDDITOR}_{POSTID}_{DATE}"
|
||||||
|
limit: 10
|
||||||
|
sort: top
|
||||||
|
subreddit:
|
||||||
|
- EarthPorn
|
||||||
|
- CityPorn
|
||||||
|
```
|
||||||
|
|
||||||
|
would be equilavent to (take note that in YAML there is `file_scheme` instead of `file-scheme`):
|
||||||
|
```bash
|
||||||
|
python3 -m bdfr download ./path/to/output --skip mp4 --skip avi --file-scheme "{UPVOTES}_{REDDITOR}_{POSTID}_{DATE}" -L 10 -S top --subreddit EarthPorn --subreddit CityPorn
|
||||||
|
```
|
||||||
|
|
||||||
|
In case when the same option is specified both in the YAML file and in as a command line argument, the command line argument takes prs
|
||||||
|
|
||||||
## Options
|
## Options
|
||||||
|
|
||||||
The following options are common between both the `archive` and `download` commands of the BDFR.
|
The following options are common between both the `archive` and `download` commands of the BDFR.
|
||||||
|
@ -80,6 +105,10 @@ The following options are common between both the `archive` and `download` comma
|
||||||
- `--config`
|
- `--config`
|
||||||
- If the path to a configuration file is supplied with this option, the BDFR will use the specified config
|
- If the path to a configuration file is supplied with this option, the BDFR will use the specified config
|
||||||
- See [Configuration Files](#configuration) for more details
|
- See [Configuration Files](#configuration) for more details
|
||||||
|
- `--opts`
|
||||||
|
- Load options from a YAML file.
|
||||||
|
- Has higher prority than the global config file but lower than command-line arguments.
|
||||||
|
- See [opts_example.yaml](./opts_example.yaml) for an example file.
|
||||||
- `--disable-module`
|
- `--disable-module`
|
||||||
- Can be specified multiple times
|
- Can be specified multiple times
|
||||||
- Disables certain modules from being used
|
- Disables certain modules from being used
|
||||||
|
@ -221,7 +250,10 @@ The `clone` command can take all the options listed above for both the `archive`
|
||||||
|
|
||||||
## Common Command Tricks
|
## Common Command Tricks
|
||||||
|
|
||||||
A common use case is for subreddits/users to be loaded from a file. The BDFR doesn't support this directly but it is simple enough to do through the command-line. Consider a list of usernames to download; they can be passed through to the BDFR with the following command, assuming that the usernames are in a text file:
|
A common use case is for subreddits/users to be loaded from a file. The BDFR supports this via YAML file options (`--opts my_opts.yaml`).
|
||||||
|
|
||||||
|
Alternatively, you can use the command-line [xargs](https://en.wikipedia.org/wiki/Xargs) function.
|
||||||
|
For a list of users `users.txt` (one user per line), type:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cat users.txt | xargs -L 1 echo --user | xargs -L 50 python3 -m bdfr download <ARGS>
|
cat users.txt | xargs -L 1 echo --user | xargs -L 50 python3 -m bdfr download <ARGS>
|
||||||
|
|
|
@ -16,6 +16,7 @@ _common_options = [
|
||||||
click.argument('directory', type=str),
|
click.argument('directory', type=str),
|
||||||
click.option('--authenticate', is_flag=True, default=None),
|
click.option('--authenticate', is_flag=True, default=None),
|
||||||
click.option('--config', type=str, default=None),
|
click.option('--config', type=str, default=None),
|
||||||
|
click.option('--opts', type=str, default=None),
|
||||||
click.option('--disable-module', multiple=True, default=None, type=str),
|
click.option('--disable-module', multiple=True, default=None, type=str),
|
||||||
click.option('--exclude-id', default=None, multiple=True),
|
click.option('--exclude-id', default=None, multiple=True),
|
||||||
click.option('--exclude-id-file', default=None, multiple=True),
|
click.option('--exclude-id-file', default=None, multiple=True),
|
||||||
|
|
|
@ -2,16 +2,21 @@
|
||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
|
|
||||||
from argparse import Namespace
|
from argparse import Namespace
|
||||||
|
from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
import logging
|
||||||
|
|
||||||
import click
|
import click
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
class Configuration(Namespace):
|
class Configuration(Namespace):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super(Configuration, self).__init__()
|
super(Configuration, self).__init__()
|
||||||
self.authenticate = False
|
self.authenticate = False
|
||||||
self.config = None
|
self.config = None
|
||||||
|
self.opts: Optional[str] = None
|
||||||
self.directory: str = '.'
|
self.directory: str = '.'
|
||||||
self.disable_module: list[str] = []
|
self.disable_module: list[str] = []
|
||||||
self.exclude_id = []
|
self.exclude_id = []
|
||||||
|
@ -49,6 +54,31 @@ class Configuration(Namespace):
|
||||||
self.comment_context: bool = False
|
self.comment_context: bool = False
|
||||||
|
|
||||||
def process_click_arguments(self, context: click.Context):
|
def process_click_arguments(self, context: click.Context):
|
||||||
|
if context.params.get('opts') is not None:
|
||||||
|
self.parse_yaml_options(context.params['opts'])
|
||||||
for arg_key in context.params.keys():
|
for arg_key in context.params.keys():
|
||||||
if arg_key in vars(self) and context.params[arg_key] is not None:
|
if not hasattr(self, arg_key):
|
||||||
vars(self)[arg_key] = context.params[arg_key]
|
logger.warning(f'Ignoring an unknown CLI argument: {arg_key}')
|
||||||
|
continue
|
||||||
|
val = context.params[arg_key]
|
||||||
|
if val is None or val == ():
|
||||||
|
# don't overwrite with an empty value
|
||||||
|
continue
|
||||||
|
setattr(self, arg_key, val)
|
||||||
|
|
||||||
|
def parse_yaml_options(self, file_path: str):
|
||||||
|
yaml_file_loc = Path(file_path)
|
||||||
|
if not yaml_file_loc.exists():
|
||||||
|
logger.error(f'No YAML file found at {yaml_file_loc}')
|
||||||
|
return
|
||||||
|
with open(yaml_file_loc) as file:
|
||||||
|
try:
|
||||||
|
opts = yaml.load(file, Loader=yaml.FullLoader)
|
||||||
|
except yaml.YAMLError as e:
|
||||||
|
logger.error(f'Could not parse YAML options file: {e}')
|
||||||
|
return
|
||||||
|
for arg_key, val in opts.items():
|
||||||
|
if not hasattr(self, arg_key):
|
||||||
|
logger.warning(f'Ignoring an unknown YAML argument: {arg_key}')
|
||||||
|
continue
|
||||||
|
setattr(self, arg_key, val)
|
||||||
|
|
9
opts_example.yaml
Normal file
9
opts_example.yaml
Normal file
|
@ -0,0 +1,9 @@
|
||||||
|
skip: [mp4, avi, mov]
|
||||||
|
file_scheme: "{UPVOTES}_{REDDITOR}_{POSTID}_{DATE}"
|
||||||
|
limit: 10
|
||||||
|
sort: top
|
||||||
|
time: all
|
||||||
|
no_dupes: true
|
||||||
|
subreddit:
|
||||||
|
- EarthPorn
|
||||||
|
- CityPorn
|
|
@ -22,3 +22,12 @@ def test_process_click_context(arg_dict: dict):
|
||||||
test_config.process_click_arguments(test_context)
|
test_config.process_click_arguments(test_context)
|
||||||
test_config = vars(test_config)
|
test_config = vars(test_config)
|
||||||
assert all([test_config[arg] == arg_dict[arg] for arg in arg_dict.keys()])
|
assert all([test_config[arg] == arg_dict[arg] for arg in arg_dict.keys()])
|
||||||
|
|
||||||
|
|
||||||
|
def test_yaml_file_read():
|
||||||
|
file = './yaml_test_configuration.yaml'
|
||||||
|
test_config = Configuration()
|
||||||
|
test_config.parse_yaml_options(file)
|
||||||
|
assert test_config.subreddit == ['EarthPorn', 'TwoXChromosomes', 'Mindustry']
|
||||||
|
assert test_config.sort == 'new'
|
||||||
|
assert test_config.limit == 10
|
||||||
|
|
6
tests/yaml_test_configuration.yaml
Normal file
6
tests/yaml_test_configuration.yaml
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
limit: 10
|
||||||
|
sort: new
|
||||||
|
subreddit:
|
||||||
|
- EarthPorn
|
||||||
|
- TwoXChromosomes
|
||||||
|
- Mindustry
|
Loading…
Reference in a new issue