lordwelch/settngs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for persistent setting groups

Browse Source 
Persistent setting groups allow settings that are not declared to
 survive normalization and other operations that would normally remove
 any unknown keys in a group.

Calling normalize or get_namespace with persistent=False will remove
 unknown keys even from persistent groups.

Currently the only way to retrieve the defaults for a config and
 preserve unknown keys is to manually get the defaults and update each
 existing group with the default values.
This commit is contained in:
Timmy Welch 2023-01-31 19:18:09 -08:00
parent c3976c2fb5
commit d2326eadb9
No known key found for this signature in database
4 changed files with 181 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
README.md
View File

@ -16,7 +16,7 @@ pip install settngs
```
A trivial example is included at the bottom of settngs.py with the output below. For a more complete example see [ComicTagger].
A trivial example is included at the bottom of settngs.py with the output below (using bash). For a more complete example see [ComicTagger].
```console
$ python -m settngs
Hello world
@ -37,6 +37,18 @@ merged_namespace.values.example_verbose=True
$ python -m settngs
Hello lordwelch
merged_namespace.values.example_verbose=True
$ cat >settings.json << EOF
{
"example": {
"hello": "lordwelch",
"verbose": true
},
"persistent": {
"test": false,
"hello": "world"
}
}
EOF
$ python -m settngs --no-verbose
Hello lordwelch
$ python -m settngs --no-verbose -s
@ -55,7 +67,9 @@ settngs.json at the end:
"example": {
"hello": "world",
"verbose": false
}
},
"persistent": false,
"hello": "world"
}
```

116
settngs.py
View File

@ -4,6 +4,7 @@ import argparse
import json
import logging
import pathlib
import re
import sys
from argparse import Namespace
from collections import defaultdict
@ -24,7 +25,14 @@ if sys.version_info < (3, 11): # pragma: no cover
else: # pragma: no cover
from typing import NamedTuple
if sys.version_info < (3, 9): # pragma: no cover
def removeprefix(self: str, prefix: str, /) -> str:
if self.startswith(prefix):
return self[len(prefix):]
else:
return self[:]
class BooleanOptionalAction(argparse.Action):
def __init__(
self,
@ -66,6 +74,7 @@ if sys.version_info < (3, 9): # pragma: no cover
setattr(namespace, self.dest, not option_string.startswith('--no-'))
else: # pragma: no cover
from argparse import BooleanOptionalAction
removeprefix = str.removeprefix
class Setting:
@ -148,7 +157,7 @@ class Setting:
for n in names:
if n.startswith('--'):
flag = True
dest_name = n.lstrip('-').replace('-', '_')
dest_name = sanitize_name(n)
break
if n.startswith('-'):
flag = True
@ -157,6 +166,8 @@ class Setting:
dest_name = names[0]
if dest:
dest_name = dest
if not dest_name.isidentifier():
raise Exception('Cannot use {dest_name} in a namespace')
internal_name = f'{prefix}_{dest_name}'.lstrip('_')
return internal_name, dest_name, flag
@ -168,8 +179,13 @@ class Setting:
return self.argparse_args, self.filter_argparse_kwargs()
class Group(NamedTuple):
persistent: bool
v: dict[str, Setting]
Values = Dict[str, Dict[str, Any]]
Definitions = Dict[str, Dict[str, Setting]]
Definitions = Dict[str, Group]
T = TypeVar('T', Values, Namespace)
@ -184,6 +200,10 @@ if TYPE_CHECKING:
ns = Namespace | Config[T] | None
def sanitize_name(name: str) -> str:
return re.sub('[' + re.escape(' -_,.!@#$%^&*(){}[]\',."<>;:') + ']+', '-', name).strip('-')
def get_option(options: Values | Namespace, setting: Setting) -> tuple[Any, bool]:
"""
Helper function to retrieve the value for a setting and if the value is the default value
@ -199,11 +219,36 @@ def get_option(options: Values | Namespace, setting: Setting) -> tuple[Any, bool
return value, value == setting.default
def get_options(options: Config[T], group: str) -> dict[str, Any]:
"""
Helper function to retrieve all of the values for a group. Only to be used on persistent groups.
Args:
options: Dictionary or namespace of options
group: The name of the group to retrieve
"""
if isinstance(options[0], dict):
values = options[0].get(group, {}).copy()
else:
internal_names = {x.internal_name: x for x in options[1][group].v.values()}
values = {}
v = vars(options[0])
for name, value in v.items():
if name.startswith(f'{group}_'):
if name in internal_names:
values[internal_names[name].dest] = value
else:
values[removeprefix(name, f'{group}_')] = value
return values
def normalize_config(
config: Config[T],
file: bool = False,
cmdline: bool = False,
defaults: bool = True,
persistent: bool = True,
) -> Config[Values]:
"""
Creates an `OptionValues` dictionary with setting definitions taken from `self.definitions`
@ -216,19 +261,23 @@ def normalize_config(
file: Include file options
cmdline: Include cmdline options
defaults: Include default values in the returned dict
raw_options_2: If set, merges non-default values into the returned dict
persistent: Include unknown keys in persistent groups
"""
normalized: Values = {}
options, definitions = config
for group_name, group in definitions.items():
group_options = {}
for setting_name, setting in group.items():
if group.persistent and persistent:
group_options = get_options(config, group_name)
for setting_name, setting in group.v.items():
if (setting.cmdline and cmdline) or (setting.file and file):
# Ensures the option exists with the default if not already set
value, default = get_option(options, setting)
if not default or default and defaults:
group_options[setting_name] = value
elif setting_name in group_options:
del group_options[setting_name]
normalized[group_name] = group_options
return Config(normalized, definitions)
@ -261,7 +310,7 @@ def clean_config(
config: Config[T], file: bool = False, cmdline: bool = False,
) -> Values:
"""
Normalizes options and then cleans up empty groups and removes 'definitions'
Normalizes options and then cleans up empty groups
Args:
options:
file:
@ -282,7 +331,7 @@ def defaults(definitions: Definitions) -> Config[Values]:
return normalize_config(Config(Namespace(), definitions), file=True, cmdline=True)
def get_namespace(config: Config[T], defaults: bool = True) -> Config[Namespace]:
def get_namespace(config: Config[T], defaults: bool = True, persistent: bool = True) -> Config[Namespace]:
"""
Returns an Namespace object with options in the form "{group_name}_{setting_name}"
`options` should already be normalized.
@ -291,6 +340,7 @@ def get_namespace(config: Config[T], defaults: bool = True) -> Config[Namespace]
Args:
options: Normalized options to turn into a Namespace
defaults: Include default values in the returned dict
persistent: Include unknown keys in persistent groups
"""
if isinstance(config.values, Namespace):
@ -299,7 +349,22 @@ def get_namespace(config: Config[T], defaults: bool = True) -> Config[Namespace]
options, definitions = config
namespace = Namespace()
for group_name, group in definitions.items():
for setting_name, setting in group.items():
if group.persistent and persistent:
group_options = get_options(config, group_name)
for name, value in group_options.items():
if name in group.v:
internal_name, default = group.v[name].internal_name, group.v[name].default == value
else:
internal_name, default = f'{group_name}_' + sanitize_name(name), None
if hasattr(namespace, internal_name):
raise Exception(f'Duplicate internal name: {internal_name}')
if not default or default and defaults:
setattr(namespace, internal_name, value)
else:
for setting_name, setting in group.v.items():
if hasattr(namespace, setting.internal_name):
raise Exception(f'Duplicate internal name: {setting.internal_name}')
value, default = get_option(options, setting)
@ -339,7 +404,7 @@ def create_argparser(definitions: Definitions, description: str, epilog: str) ->
description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter,
)
for group_name, group in definitions.items():
for setting_name, setting in group.items():
for setting_name, setting in group.v.items():
if setting.cmdline:
argparse_args, argparse_kwargs = setting.to_argparse()
current_group: ArgParser = argparser
@ -416,7 +481,7 @@ class Manager:
if isinstance(definitions, Config):
self.definitions = definitions.definitions
else:
self.definitions = defaultdict(lambda: dict(), definitions or {})
self.definitions = defaultdict(lambda: Group(False, {}), definitions or {})
self.exclusive_group = False
self.current_group_name = ''
@ -427,20 +492,36 @@ class Manager:
def add_setting(self, *args: Any, **kwargs: Any) -> None:
"""Takes passes all arguments through to `Setting`, `group` and `exclusive` are already set"""
setting = Setting(*args, **kwargs, group=self.current_group_name, exclusive=self.exclusive_group)
self.definitions[self.current_group_name][setting.dest] = setting
self.definitions[self.current_group_name].v[setting.dest] = setting
def add_group(self, name: str, add_settings: Callable[[Manager], None], exclusive_group: bool = False) -> None:
def add_group(self, name: str, group: Callable[[Manager], None], exclusive_group: bool = False) -> None:
"""
The primary way to add define options on this class
Args:
name: The name of the group to define
add_settings: A function that registers individual options using :meth:`add_setting`
group: A function that registers individual options using :meth:`add_setting`
exclusive_group: If this group is an argparse exclusive group
"""
self.current_group_name = name
self.exclusive_group = exclusive_group
add_settings(self)
group(self)
self.current_group_name = ''
self.exclusive_group = False
def add_persistent_group(self, name: str, group: Callable[[Manager], None], exclusive_group: bool = False) -> None:
"""
The primary way to add define options on this class
Args:
name: The name of the group to define
group: A function that registers individual options using :meth:`add_setting`
exclusive_group: If this group is an argparse exclusive group
"""
self.current_group_name = name
self.exclusive_group = exclusive_group
self.definitions[self.current_group_name] = Group(True, {})
group(self)
self.current_group_name = ''
self.exclusive_group = False
@ -527,11 +608,20 @@ def example(manager: Manager) -> None:
)
def persistent(manager: Manager) -> None:
manager.add_setting(
'--test', '-t',
default=False,
action=BooleanOptionalAction, # Added in Python 3.9
)
def _main(args: list[str] | None = None) -> None:
settings_path = pathlib.Path('./settings.json')
manager = Manager(description='This is an example', epilog='goodbye!')
manager.add_group('example', example)
manager.add_persistent_group('persistent', persistent)
file_config, success = manager.parse_file(settings_path)
file_namespace = manager.get_namespace(file_config)

29
testing/settngs.py
View File

@ -15,47 +15,52 @@ example: list[tuple[list[str], str, str]] = [
(
['--hello', 'lordwelch', '-s'],
'Hello lordwelch\nSuccessfully saved settings to settings.json\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n }\n}\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n },\n "persistent": {\n "test": false\n }\n}\n',
),
(
[],
'Hello lordwelch\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n }\n}\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n },\n "persistent": {\n "test": false\n }\n}\n',
),
(
['-v'],
'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n }\n}\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n },\n "persistent": {\n "test": false\n }\n}\n',
),
(
['-v', '-s'],
'Hello lordwelch\nSuccessfully saved settings to settings.json\nmerged_namespace.values.example_verbose=True\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n }\n}\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false\n }\n}\n',
),
(
[],
'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n }\n}\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false\n }\n}\n',
),
(
['--no-verbose'],
['manual settings.json'],
'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n',
),
(
['--no-verbose', '-t'],
'Hello lordwelch\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n }\n}\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n',
),
(
['--no-verbose', '-s'],
['--no-verbose', '-s', '-t'],
'Hello lordwelch\nSuccessfully saved settings to settings.json\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n }\n}\n',
'{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n },\n "persistent": {\n "test": true,\n "hello": "world"\n }\n}\n',
),
(
['--hello', 'world', '--no-verbose', '-s'],
['--hello', 'world', '--no-verbose', '--no-test', '-s'],
'Hello world\nSuccessfully saved settings to settings.json\n',
'{\n "example": {\n "hello": "world",\n "verbose": false\n }\n}\n',
'{\n "example": {\n "hello": "world",\n "verbose": false\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n',
),
(
[],
'Hello world\n',
'{\n "example": {\n "hello": "world",\n "verbose": false\n }\n}\n',
'{\n "example": {\n "hello": "world",\n "verbose": false\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n',
),
]
success = [

51
tests/settngs_test.py
View File

@ -6,6 +6,7 @@ import json
import pytest
import settngs
from settngs import Group
from testing.settngs import example
from testing.settngs import failure
from testing.settngs import success
@ -27,7 +28,7 @@ def test_settngs_manager_config():
manager = settngs.Manager(
definitions=settngs.Config[settngs.Namespace](
settngs.Namespace(),
{'tst': {'test': settngs.Setting('--test', default='hello', group='tst', exclusive=False)}},
{'tst': Group(False, {'test': settngs.Setting('--test', default='hello', group='tst', exclusive=False)})},
),
)
@ -97,12 +98,15 @@ def test_cmdline_only(settngs_manager):
def test_normalize(settngs_manager):
settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
defaults = settngs_manager.defaults()
defaults.values['test'] = 'fail' # Not defined in settngs_manager
defaults.values['test'] = 'fail' # Not defined in settngs_manager, should be removed
defaults.values['persistent']['hello'] = 'success' # Not defined in settngs_manager, should stay
defaults_namespace = settngs_manager.get_namespace(settngs_manager.defaults())
defaults_namespace.values.test = 'fail'
defaults_namespace.values.test = 'fail' # Not defined in settngs_manager, should be removed
defaults_namespace.values.persistent_hello = 'success' # Not defined in settngs_manager, should stay
normalized, _ = settngs_manager.normalize_config(defaults, file=True)
normalized_from_namespace = settngs_manager.normalize_config(defaults_namespace, file=True)
@ -112,17 +116,21 @@ def test_normalize(settngs_manager):
assert 'tst' in normalized
assert 'test' in normalized['tst']
assert normalized['tst']['test'] == 'hello'
assert normalized['persistent']['hello'] == 'success'
assert not hasattr(normalized_namespace, 'test')
assert hasattr(normalized_namespace, 'tst_test')
assert normalized_namespace.tst_test == 'hello'
assert normalized_namespace.persistent_hello == 'success'
def test_clean_config(settngs_manager):
settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
settngs_manager.add_group('tst2', lambda parser: parser.add_setting('--test2', default='hello', file=False))
settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
normalized, _ = settngs_manager.defaults()
normalized['tst']['test'] = 'success'
normalized['persistent']['hello'] = 'success'
normalized['fail'] = 'fail'
cleaned = settngs_manager.clean_config(normalized, file=True)
@ -130,6 +138,7 @@ def test_clean_config(settngs_manager):
assert 'fail' not in cleaned
assert 'tst2' not in cleaned
assert cleaned['tst']['test'] == 'success'
assert cleaned['persistent']['hello'] == 'success'
def test_parse_cmdline(settngs_manager, tmp_path):
@ -154,14 +163,16 @@ def test_parse_cmdline_with_namespace(settngs_manager, tmp_path):
def test_parse_file(settngs_manager, tmp_path):
settngs_file = tmp_path / 'settngs.json'
settngs_file.write_text(json.dumps({'tst': {'test': 'success'}}))
settngs_file.write_text(json.dumps({'tst': {'test': 'success'}, 'persistent': {'hello': 'success'}}))
settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
normalized, success = settngs_manager.parse_file(settngs_file)
assert success
assert 'test' in normalized[0]['tst']
assert normalized[0]['tst']['test'] == 'success'
assert normalized[0]['persistent']['hello'] == 'success'
def test_parse_non_existent_file(settngs_manager, tmp_path):
@ -190,15 +201,18 @@ def test_parse_corrupt_file(settngs_manager, tmp_path):
def test_save_file(settngs_manager, tmp_path):
settngs_file = tmp_path / 'settngs.json'
settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
normalized, _ = settngs_manager.defaults()
normalized['tst']['test'] = 'success'
normalized['persistent']['hello'] = 'success'
success = settngs_manager.save_file(normalized, settngs_file)
normalized, success_r = settngs_manager.parse_file(settngs_file)
normalized_r, success_r = settngs_manager.parse_file(settngs_file)
assert success and success_r
assert 'test' in normalized[0]['tst']
assert normalized[0]['tst']['test'] == 'success'
assert 'test' in normalized_r[0]['tst']
assert normalized_r[0]['tst']['test'] == 'success'
assert normalized_r[0]['persistent']['hello'] == 'success'
def test_save_file_not_seriazable(settngs_manager, tmp_path):
@ -208,11 +222,16 @@ def test_save_file_not_seriazable(settngs_manager, tmp_path):
normalized['tst']['test'] = {'fail'} # Sets are not serializabl
success = settngs_manager.save_file(normalized, settngs_file)
normalized, success_r = settngs_manager.parse_file(settngs_file)
normalized_r, success_r = settngs_manager.parse_file(settngs_file)
# normalized_r will be the default settings
assert not (success and success_r)
assert 'test' in normalized[0]['tst']
assert normalized[0]['tst']['test'] == 'hello'
assert not success
assert not success_r
assert 'test' in normalized['tst']
assert normalized['tst']['test'] == {'fail'}
assert 'test' in normalized_r[0]['tst']
assert normalized_r[0]['tst']['test'] == 'hello'
def test_cli_set(settngs_manager, tmp_path):
@ -272,8 +291,14 @@ def test_example(capsys, tmp_path, monkeypatch):
settings_file = tmp_path / 'settings.json'
settings_file.touch()
i = 0
for args, expected_out, expected_file in example:
if args == ['manual settings.json']:
settings_file.unlink()
settings_file.write_text('{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n')
else:
settngs._main(args)
captured = capsys.readouterr()
assert captured.out == expected_out, args
assert settings_file.read_text() == expected_file, args
assert captured.out == expected_out, f'{i}, {args}'
assert settings_file.read_text() == expected_file, f'{i}, {args}'
i += 1