From d2326eadb9e3f5bc6c292a4c34e19a50de5866be Mon Sep 17 00:00:00 2001 From: Timmy Welch Date: Tue, 31 Jan 2023 19:18:09 -0800 Subject: [PATCH] Add support for persistent setting groups Persistent setting groups allow settings that are not declared to survive normalization and other operations that would normally remove any unknown keys in a group. Calling normalize or get_namespace with persistent=False will remove unknown keys even from persistent groups. Currently the only way to retrieve the defaults for a config and preserve unknown keys is to manually get the defaults and update each existing group with the default values. --- README.md | 18 +++++- settngs.py | 126 ++++++++++++++++++++++++++++++++++++------ testing/settngs.py | 29 ++++++---- tests/settngs_test.py | 55 +++++++++++++----- 4 files changed, 181 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 6f9fa69..6c42498 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ pip install settngs ``` -A trivial example is included at the bottom of settngs.py with the output below. For a more complete example see [ComicTagger]. +A trivial example is included at the bottom of settngs.py with the output below (using bash). For a more complete example see [ComicTagger]. ```console $ python -m settngs Hello world @@ -37,6 +37,18 @@ merged_namespace.values.example_verbose=True $ python -m settngs Hello lordwelch merged_namespace.values.example_verbose=True +$ cat >settings.json << EOF +{ + "example": { + "hello": "lordwelch", + "verbose": true + }, + "persistent": { + "test": false, + "hello": "world" + } +} +EOF $ python -m settngs --no-verbose Hello lordwelch $ python -m settngs --no-verbose -s @@ -55,7 +67,9 @@ settngs.json at the end: "example": { "hello": "world", "verbose": false - } + }, + "persistent": false, + "hello": "world" } ``` diff --git a/settngs.py b/settngs.py index b94cf15..4f4243d 100644 --- a/settngs.py +++ b/settngs.py @@ -4,6 +4,7 @@ import argparse import json import logging import pathlib +import re import sys from argparse import Namespace from collections import defaultdict @@ -24,7 +25,14 @@ if sys.version_info < (3, 11): # pragma: no cover else: # pragma: no cover from typing import NamedTuple + if sys.version_info < (3, 9): # pragma: no cover + def removeprefix(self: str, prefix: str, /) -> str: + if self.startswith(prefix): + return self[len(prefix):] + else: + return self[:] + class BooleanOptionalAction(argparse.Action): def __init__( self, @@ -66,6 +74,7 @@ if sys.version_info < (3, 9): # pragma: no cover setattr(namespace, self.dest, not option_string.startswith('--no-')) else: # pragma: no cover from argparse import BooleanOptionalAction + removeprefix = str.removeprefix class Setting: @@ -148,7 +157,7 @@ class Setting: for n in names: if n.startswith('--'): flag = True - dest_name = n.lstrip('-').replace('-', '_') + dest_name = sanitize_name(n) break if n.startswith('-'): flag = True @@ -157,6 +166,8 @@ class Setting: dest_name = names[0] if dest: dest_name = dest + if not dest_name.isidentifier(): + raise Exception('Cannot use {dest_name} in a namespace') internal_name = f'{prefix}_{dest_name}'.lstrip('_') return internal_name, dest_name, flag @@ -168,8 +179,13 @@ class Setting: return self.argparse_args, self.filter_argparse_kwargs() +class Group(NamedTuple): + persistent: bool + v: dict[str, Setting] + + Values = Dict[str, Dict[str, Any]] -Definitions = Dict[str, Dict[str, Setting]] +Definitions = Dict[str, Group] T = TypeVar('T', Values, Namespace) @@ -184,6 +200,10 @@ if TYPE_CHECKING: ns = Namespace | Config[T] | None +def sanitize_name(name: str) -> str: + return re.sub('[' + re.escape(' -_,.!@#$%^&*(){}[]\',."<>;:') + ']+', '-', name).strip('-') + + def get_option(options: Values | Namespace, setting: Setting) -> tuple[Any, bool]: """ Helper function to retrieve the value for a setting and if the value is the default value @@ -199,11 +219,36 @@ def get_option(options: Values | Namespace, setting: Setting) -> tuple[Any, bool return value, value == setting.default +def get_options(options: Config[T], group: str) -> dict[str, Any]: + """ + Helper function to retrieve all of the values for a group. Only to be used on persistent groups. + + Args: + options: Dictionary or namespace of options + group: The name of the group to retrieve + """ + if isinstance(options[0], dict): + values = options[0].get(group, {}).copy() + else: + internal_names = {x.internal_name: x for x in options[1][group].v.values()} + values = {} + v = vars(options[0]) + for name, value in v.items(): + if name.startswith(f'{group}_'): + if name in internal_names: + values[internal_names[name].dest] = value + else: + values[removeprefix(name, f'{group}_')] = value + + return values + + def normalize_config( config: Config[T], file: bool = False, cmdline: bool = False, defaults: bool = True, + persistent: bool = True, ) -> Config[Values]: """ Creates an `OptionValues` dictionary with setting definitions taken from `self.definitions` @@ -216,19 +261,23 @@ def normalize_config( file: Include file options cmdline: Include cmdline options defaults: Include default values in the returned dict - raw_options_2: If set, merges non-default values into the returned dict + persistent: Include unknown keys in persistent groups """ normalized: Values = {} options, definitions = config for group_name, group in definitions.items(): group_options = {} - for setting_name, setting in group.items(): + if group.persistent and persistent: + group_options = get_options(config, group_name) + for setting_name, setting in group.v.items(): if (setting.cmdline and cmdline) or (setting.file and file): # Ensures the option exists with the default if not already set value, default = get_option(options, setting) if not default or default and defaults: group_options[setting_name] = value + elif setting_name in group_options: + del group_options[setting_name] normalized[group_name] = group_options return Config(normalized, definitions) @@ -261,7 +310,7 @@ def clean_config( config: Config[T], file: bool = False, cmdline: bool = False, ) -> Values: """ - Normalizes options and then cleans up empty groups and removes 'definitions' + Normalizes options and then cleans up empty groups Args: options: file: @@ -282,7 +331,7 @@ def defaults(definitions: Definitions) -> Config[Values]: return normalize_config(Config(Namespace(), definitions), file=True, cmdline=True) -def get_namespace(config: Config[T], defaults: bool = True) -> Config[Namespace]: +def get_namespace(config: Config[T], defaults: bool = True, persistent: bool = True) -> Config[Namespace]: """ Returns an Namespace object with options in the form "{group_name}_{setting_name}" `options` should already be normalized. @@ -291,6 +340,7 @@ def get_namespace(config: Config[T], defaults: bool = True) -> Config[Namespace] Args: options: Normalized options to turn into a Namespace defaults: Include default values in the returned dict + persistent: Include unknown keys in persistent groups """ if isinstance(config.values, Namespace): @@ -299,13 +349,28 @@ def get_namespace(config: Config[T], defaults: bool = True) -> Config[Namespace] options, definitions = config namespace = Namespace() for group_name, group in definitions.items(): - for setting_name, setting in group.items(): - if hasattr(namespace, setting.internal_name): - raise Exception(f'Duplicate internal name: {setting.internal_name}') - value, default = get_option(options, setting) + if group.persistent and persistent: + group_options = get_options(config, group_name) + for name, value in group_options.items(): + if name in group.v: + internal_name, default = group.v[name].internal_name, group.v[name].default == value + else: + internal_name, default = f'{group_name}_' + sanitize_name(name), None - if not default or default and defaults: - setattr(namespace, setting.internal_name, value) + if hasattr(namespace, internal_name): + raise Exception(f'Duplicate internal name: {internal_name}') + + if not default or default and defaults: + setattr(namespace, internal_name, value) + + else: + for setting_name, setting in group.v.items(): + if hasattr(namespace, setting.internal_name): + raise Exception(f'Duplicate internal name: {setting.internal_name}') + value, default = get_option(options, setting) + + if not default or default and defaults: + setattr(namespace, setting.internal_name, value) return Config(namespace, definitions) @@ -339,7 +404,7 @@ def create_argparser(definitions: Definitions, description: str, epilog: str) -> description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter, ) for group_name, group in definitions.items(): - for setting_name, setting in group.items(): + for setting_name, setting in group.v.items(): if setting.cmdline: argparse_args, argparse_kwargs = setting.to_argparse() current_group: ArgParser = argparser @@ -416,7 +481,7 @@ class Manager: if isinstance(definitions, Config): self.definitions = definitions.definitions else: - self.definitions = defaultdict(lambda: dict(), definitions or {}) + self.definitions = defaultdict(lambda: Group(False, {}), definitions or {}) self.exclusive_group = False self.current_group_name = '' @@ -427,20 +492,36 @@ class Manager: def add_setting(self, *args: Any, **kwargs: Any) -> None: """Takes passes all arguments through to `Setting`, `group` and `exclusive` are already set""" setting = Setting(*args, **kwargs, group=self.current_group_name, exclusive=self.exclusive_group) - self.definitions[self.current_group_name][setting.dest] = setting + self.definitions[self.current_group_name].v[setting.dest] = setting - def add_group(self, name: str, add_settings: Callable[[Manager], None], exclusive_group: bool = False) -> None: + def add_group(self, name: str, group: Callable[[Manager], None], exclusive_group: bool = False) -> None: """ The primary way to add define options on this class Args: name: The name of the group to define - add_settings: A function that registers individual options using :meth:`add_setting` + group: A function that registers individual options using :meth:`add_setting` exclusive_group: If this group is an argparse exclusive group """ self.current_group_name = name self.exclusive_group = exclusive_group - add_settings(self) + group(self) + self.current_group_name = '' + self.exclusive_group = False + + def add_persistent_group(self, name: str, group: Callable[[Manager], None], exclusive_group: bool = False) -> None: + """ + The primary way to add define options on this class + + Args: + name: The name of the group to define + group: A function that registers individual options using :meth:`add_setting` + exclusive_group: If this group is an argparse exclusive group + """ + self.current_group_name = name + self.exclusive_group = exclusive_group + self.definitions[self.current_group_name] = Group(True, {}) + group(self) self.current_group_name = '' self.exclusive_group = False @@ -527,11 +608,20 @@ def example(manager: Manager) -> None: ) +def persistent(manager: Manager) -> None: + manager.add_setting( + '--test', '-t', + default=False, + action=BooleanOptionalAction, # Added in Python 3.9 + ) + + def _main(args: list[str] | None = None) -> None: settings_path = pathlib.Path('./settings.json') manager = Manager(description='This is an example', epilog='goodbye!') manager.add_group('example', example) + manager.add_persistent_group('persistent', persistent) file_config, success = manager.parse_file(settings_path) file_namespace = manager.get_namespace(file_config) diff --git a/testing/settngs.py b/testing/settngs.py index 28bd509..3327309 100644 --- a/testing/settngs.py +++ b/testing/settngs.py @@ -15,47 +15,52 @@ example: list[tuple[list[str], str, str]] = [ ( ['--hello', 'lordwelch', '-s'], 'Hello lordwelch\nSuccessfully saved settings to settings.json\n', - '{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n }\n}\n', + '{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n },\n "persistent": {\n "test": false\n }\n}\n', ), ( [], 'Hello lordwelch\n', - '{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n }\n}\n', + '{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n },\n "persistent": {\n "test": false\n }\n}\n', ), ( ['-v'], 'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n', - '{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n }\n}\n', + '{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n },\n "persistent": {\n "test": false\n }\n}\n', ), ( ['-v', '-s'], 'Hello lordwelch\nSuccessfully saved settings to settings.json\nmerged_namespace.values.example_verbose=True\n', - '{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n }\n}\n', + '{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false\n }\n}\n', ), ( [], 'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n', - '{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n }\n}\n', + '{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false\n }\n}\n', ), ( - ['--no-verbose'], + ['manual settings.json'], + 'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n', + '{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n', + ), + ( + ['--no-verbose', '-t'], 'Hello lordwelch\n', - '{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n }\n}\n', + '{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n', ), ( - ['--no-verbose', '-s'], + ['--no-verbose', '-s', '-t'], 'Hello lordwelch\nSuccessfully saved settings to settings.json\n', - '{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n }\n}\n', + '{\n "example": {\n "hello": "lordwelch",\n "verbose": false\n },\n "persistent": {\n "test": true,\n "hello": "world"\n }\n}\n', ), ( - ['--hello', 'world', '--no-verbose', '-s'], + ['--hello', 'world', '--no-verbose', '--no-test', '-s'], 'Hello world\nSuccessfully saved settings to settings.json\n', - '{\n "example": {\n "hello": "world",\n "verbose": false\n }\n}\n', + '{\n "example": {\n "hello": "world",\n "verbose": false\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n', ), ( [], 'Hello world\n', - '{\n "example": {\n "hello": "world",\n "verbose": false\n }\n}\n', + '{\n "example": {\n "hello": "world",\n "verbose": false\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n', ), ] success = [ diff --git a/tests/settngs_test.py b/tests/settngs_test.py index 18a5cd0..bc696b3 100644 --- a/tests/settngs_test.py +++ b/tests/settngs_test.py @@ -6,6 +6,7 @@ import json import pytest import settngs +from settngs import Group from testing.settngs import example from testing.settngs import failure from testing.settngs import success @@ -27,7 +28,7 @@ def test_settngs_manager_config(): manager = settngs.Manager( definitions=settngs.Config[settngs.Namespace]( settngs.Namespace(), - {'tst': {'test': settngs.Setting('--test', default='hello', group='tst', exclusive=False)}}, + {'tst': Group(False, {'test': settngs.Setting('--test', default='hello', group='tst', exclusive=False)})}, ), ) @@ -97,12 +98,15 @@ def test_cmdline_only(settngs_manager): def test_normalize(settngs_manager): settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello')) + settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world')) defaults = settngs_manager.defaults() - defaults.values['test'] = 'fail' # Not defined in settngs_manager + defaults.values['test'] = 'fail' # Not defined in settngs_manager, should be removed + defaults.values['persistent']['hello'] = 'success' # Not defined in settngs_manager, should stay defaults_namespace = settngs_manager.get_namespace(settngs_manager.defaults()) - defaults_namespace.values.test = 'fail' + defaults_namespace.values.test = 'fail' # Not defined in settngs_manager, should be removed + defaults_namespace.values.persistent_hello = 'success' # Not defined in settngs_manager, should stay normalized, _ = settngs_manager.normalize_config(defaults, file=True) normalized_from_namespace = settngs_manager.normalize_config(defaults_namespace, file=True) @@ -112,17 +116,21 @@ def test_normalize(settngs_manager): assert 'tst' in normalized assert 'test' in normalized['tst'] assert normalized['tst']['test'] == 'hello' + assert normalized['persistent']['hello'] == 'success' assert not hasattr(normalized_namespace, 'test') assert hasattr(normalized_namespace, 'tst_test') assert normalized_namespace.tst_test == 'hello' + assert normalized_namespace.persistent_hello == 'success' def test_clean_config(settngs_manager): settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False)) settngs_manager.add_group('tst2', lambda parser: parser.add_setting('--test2', default='hello', file=False)) + settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world')) normalized, _ = settngs_manager.defaults() normalized['tst']['test'] = 'success' + normalized['persistent']['hello'] = 'success' normalized['fail'] = 'fail' cleaned = settngs_manager.clean_config(normalized, file=True) @@ -130,6 +138,7 @@ def test_clean_config(settngs_manager): assert 'fail' not in cleaned assert 'tst2' not in cleaned assert cleaned['tst']['test'] == 'success' + assert cleaned['persistent']['hello'] == 'success' def test_parse_cmdline(settngs_manager, tmp_path): @@ -154,14 +163,16 @@ def test_parse_cmdline_with_namespace(settngs_manager, tmp_path): def test_parse_file(settngs_manager, tmp_path): settngs_file = tmp_path / 'settngs.json' - settngs_file.write_text(json.dumps({'tst': {'test': 'success'}})) + settngs_file.write_text(json.dumps({'tst': {'test': 'success'}, 'persistent': {'hello': 'success'}})) settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False)) + settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world')) normalized, success = settngs_manager.parse_file(settngs_file) assert success assert 'test' in normalized[0]['tst'] assert normalized[0]['tst']['test'] == 'success' + assert normalized[0]['persistent']['hello'] == 'success' def test_parse_non_existent_file(settngs_manager, tmp_path): @@ -190,15 +201,18 @@ def test_parse_corrupt_file(settngs_manager, tmp_path): def test_save_file(settngs_manager, tmp_path): settngs_file = tmp_path / 'settngs.json' settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False)) + settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world')) normalized, _ = settngs_manager.defaults() normalized['tst']['test'] = 'success' + normalized['persistent']['hello'] = 'success' success = settngs_manager.save_file(normalized, settngs_file) - normalized, success_r = settngs_manager.parse_file(settngs_file) + normalized_r, success_r = settngs_manager.parse_file(settngs_file) assert success and success_r - assert 'test' in normalized[0]['tst'] - assert normalized[0]['tst']['test'] == 'success' + assert 'test' in normalized_r[0]['tst'] + assert normalized_r[0]['tst']['test'] == 'success' + assert normalized_r[0]['persistent']['hello'] == 'success' def test_save_file_not_seriazable(settngs_manager, tmp_path): @@ -208,11 +222,16 @@ def test_save_file_not_seriazable(settngs_manager, tmp_path): normalized['tst']['test'] = {'fail'} # Sets are not serializabl success = settngs_manager.save_file(normalized, settngs_file) - normalized, success_r = settngs_manager.parse_file(settngs_file) + normalized_r, success_r = settngs_manager.parse_file(settngs_file) + # normalized_r will be the default settings - assert not (success and success_r) - assert 'test' in normalized[0]['tst'] - assert normalized[0]['tst']['test'] == 'hello' + assert not success + assert not success_r + assert 'test' in normalized['tst'] + assert normalized['tst']['test'] == {'fail'} + + assert 'test' in normalized_r[0]['tst'] + assert normalized_r[0]['tst']['test'] == 'hello' def test_cli_set(settngs_manager, tmp_path): @@ -272,8 +291,14 @@ def test_example(capsys, tmp_path, monkeypatch): settings_file = tmp_path / 'settings.json' settings_file.touch() + i = 0 for args, expected_out, expected_file in example: - settngs._main(args) - captured = capsys.readouterr() - assert captured.out == expected_out, args - assert settings_file.read_text() == expected_file, args + if args == ['manual settings.json']: + settings_file.unlink() + settings_file.write_text('{\n "example": {\n "hello": "lordwelch",\n "verbose": true\n },\n "persistent": {\n "test": false,\n "hello": "world"\n }\n}\n') + else: + settngs._main(args) + captured = capsys.readouterr() + assert captured.out == expected_out, f'{i}, {args}' + assert settings_file.read_text() == expected_file, f'{i}, {args}' + i += 1