Version Bump

Persistent setting groups allow settings that are not declared to
 survive normalization and other operations that would normally remove
 any unknown keys in a group.

Calling normalize or get_namespace with persistent=False will remove
 unknown keys even from persistent groups.

Currently the only way to retrieve the defaults for a config and
 preserve unknown keys is to manually get the defaults and update each
 existing group with the default values.

README.md

@@ -16,7 +16,7 @@ pip install settngs
 ```
 
 
-A trivial example is included at the bottom of settngs.py with the output below. For a more complete example see [ComicTagger].
+A trivial example is included at the bottom of settngs.py with the output below (using bash). For a more complete example see [ComicTagger].
 ```console
 $ python -m settngs
 Hello world
@@ -24,27 +24,39 @@ $ python -m settngs --hello lordwelch
 Hello lordwelch
 $ python -m settngs --hello lordwelch -s
 Hello lordwelch
-Successfully saved settngs to settngs.json
+Successfully saved settings to settings.json
 $ python -m settngs
 Hello lordwelch
 $ python -m settngs -v
 Hello lordwelch
-merged_namespace.example_verbose=True
+merged_namespace.values.example_verbose=True
 $ python -m settngs -v -s
 Hello lordwelch
-Successfully saved settngs to settngs.json
-merged_namespace.example_verbose=True
+Successfully saved settings to settings.json
+merged_namespace.values.example_verbose=True
 $ python -m settngs
 Hello lordwelch
-merged_namespace.example_verbose=True
+merged_namespace.values.example_verbose=True
+$ cat >settings.json << EOF
+{
+  "example": {
+    "hello": "lordwelch",
+    "verbose": true
+  },
+  "persistent": {
+    "test": false,
+    "hello": "world"
+  }
+}
+EOF
 $ python -m settngs --no-verbose
 Hello lordwelch
 $ python -m settngs --no-verbose -s
 Hello lordwelch
-Successfully saved settngs to settngs.json
+Successfully saved settings to settings.json
 $ python -m settngs --hello world --no-verbose -s
 Hello world
-Successfully saved settngs to settngs.json
+Successfully saved settings to settings.json
 $ python -m settngs
 Hello world
 ```
@@ -55,7 +67,9 @@ settngs.json at the end:
   "example": {
     "hello": "world",
     "verbose": false
-  }
+  },
+  "persistent": false,
+  "hello": "world"
 }
 ```
 

settngs.py

@@ -4,18 +4,78 @@ import argparse
 import json
 import logging
 import pathlib
+import re
+import sys
+from argparse import Namespace
 from collections import defaultdict
 from collections.abc import Sequence
 from typing import Any
 from typing import Callable
 from typing import Dict
-from typing import NamedTuple
+from typing import Generic
 from typing import NoReturn
+from typing import overload
 from typing import TYPE_CHECKING
+from typing import TypeVar
 from typing import Union
-
 logger = logging.getLogger(__name__)
 
+if sys.version_info < (3, 11):  # pragma: no cover
+    from typing_extensions import NamedTuple
+else:  # pragma: no cover
+    from typing import NamedTuple
+
+
+if sys.version_info < (3, 9):  # pragma: no cover
+    def removeprefix(self: str, prefix: str, /) -> str:
+        if self.startswith(prefix):
+            return self[len(prefix):]
+        else:
+            return self[:]
+
+    class BooleanOptionalAction(argparse.Action):
+        def __init__(
+            self,
+            option_strings,
+            dest,
+            default=None,
+            type=None,  # noqa: A002
+            choices=None,
+            required=False,
+            help=None,  # noqa: A002
+            metavar=None,
+        ):
+
+            _option_strings = []
+            for option_string in option_strings:
+                _option_strings.append(option_string)
+
+                if option_string.startswith('--'):
+                    option_string = '--no-' + option_string[2:]
+                    _option_strings.append(option_string)
+
+            if help is not None and default is not None and default is not argparse.SUPPRESS:
+                help += ' (default: %(default)s)'
+
+            super().__init__(
+                option_strings=_option_strings,
+                dest=dest,
+                nargs=0,
+                default=default,
+                type=type,
+                choices=choices,
+                required=required,
+                help=help,
+                metavar=metavar,
+            )
+
+        def __call__(self, parser, namespace, values, option_string=None):  # dead: disable
+            if option_string in self.option_strings:
+                setattr(namespace, self.dest, not option_string.startswith('--no-'))
+else:  # pragma: no cover
+    from argparse import BooleanOptionalAction
+    removeprefix = str.removeprefix
+
 
 class Setting:
     def __init__(
@@ -84,10 +144,10 @@ class Setting:
             'dest': self.internal_name if flag else None,
         }
 
-    def __str__(self) -> str:
+    def __str__(self) -> str:  # pragma: no cover
         return f'Setting({self.argparse_args}, type={self.type}, file={self.file}, cmdline={self.cmdline}, kwargs={self.argparse_kwargs})'
 
-    def __repr__(self) -> str:
+    def __repr__(self) -> str:  # pragma: no cover
         return self.__str__()
 
     def get_dest(self, prefix: str, names: Sequence[str], dest: str | None) -> tuple[str, str, bool]:
@@ -97,7 +157,7 @@ class Setting:
         for n in names:
             if n.startswith('--'):
                 flag = True
-                dest_name = n.lstrip('-').replace('-', '_')
+                dest_name = sanitize_name(n)
                 break
             if n.startswith('-'):
                 flag = True
@@ -106,6 +166,8 @@ class Setting:
             dest_name = names[0]
         if dest:
             dest_name = dest
+        if not dest_name.isidentifier():
+            raise Exception('Cannot use {dest_name} in a namespace')
 
         internal_name = f'{prefix}_{dest_name}'.lstrip('_')
         return internal_name, dest_name, flag
@@ -117,20 +179,32 @@ class Setting:
         return self.argparse_args, self.filter_argparse_kwargs()
 
 
+class Group(NamedTuple):
+    persistent: bool
+    v: dict[str, Setting]
+
+
 Values = Dict[str, Dict[str, Any]]
-Definitions = Dict[str, Dict[str, Setting]]
+Definitions = Dict[str, Group]
+
+T = TypeVar('T', Values, Namespace)
 
 
-class Config(NamedTuple):
-    values: Values
+class Config(NamedTuple, Generic[T]):
+    values: T
     definitions: Definitions
 
 
 if TYPE_CHECKING:
     ArgParser = Union[argparse._MutuallyExclusiveGroup, argparse._ArgumentGroup, argparse.ArgumentParser]
+    ns = Namespace | Config[T] | None
 
 
-def get_option(options: Values | argparse.Namespace, setting: Setting) -> tuple[Any, bool]:
+def sanitize_name(name: str) -> str:
+    return re.sub('[' + re.escape(' -_,.!@#$%^&*(){}[]\',."<>;:') + ']+', '_', name).strip('_')
+
+
+def get_option(options: Values | Namespace, setting: Setting) -> tuple[Any, bool]:
     """
     Helper function to retrieve the value for a setting and if the value is the default value
 
@@ -145,14 +219,37 @@ def get_option(options: Values | argparse.Namespace, setting: Setting) -> tuple[
     return value, value == setting.default
 
 
+def get_options(options: Config[T], group: str) -> dict[str, Any]:
+    """
+    Helper function to retrieve all of the values for a group. Only to be used on persistent groups.
+
+    Args:
+        options: Dictionary or namespace of options
+        group: The name of the group to retrieve
+    """
+    if isinstance(options[0], dict):
+        values = options[0].get(group, {}).copy()
+    else:
+        internal_names = {x.internal_name: x for x in options[1][group].v.values()}
+        values = {}
+        v = vars(options[0])
+        for name, value in v.items():
+            if name.startswith(f'{group}_'):
+                if name in internal_names:
+                    values[internal_names[name].dest] = value
+                else:
+                    values[removeprefix(name, f'{group}_')] = value
+
+    return values
+
+
 def normalize_config(
-    raw_options: Values | argparse.Namespace,
-    definitions: Definitions,
+    config: Config[T],
     file: bool = False,
     cmdline: bool = False,
     defaults: bool = True,
-    raw_options_2: Values | argparse.Namespace | None = None,
-) -> Values:
+    persistent: bool = True,
+) -> Config[Values]:
     """
     Creates an `OptionValues` dictionary with setting definitions taken from `self.definitions`
     and values taken from `raw_options` and `raw_options_2' if defined.
@@ -164,29 +261,28 @@ def normalize_config(
         file: Include file options
         cmdline: Include cmdline options
         defaults: Include default values in the returned dict
-        raw_options_2: If set, merges non-default values into the returned dict
+        persistent: Include unknown keys in persistent groups
     """
-    options: Values = {}
+
+    normalized: Values = {}
+    options, definitions = config
     for group_name, group in definitions.items():
         group_options = {}
-        for setting_name, setting in group.items():
+        if group.persistent and persistent:
+            group_options = get_options(config, group_name)
+        for setting_name, setting in group.v.items():
             if (setting.cmdline and cmdline) or (setting.file and file):
                 # Ensures the option exists with the default if not already set
-                value, default = get_option(raw_options, setting)
+                value, default = get_option(options, setting)
                 if not default or default and defaults:
                     group_options[setting_name] = value
-
-                # will override with option from raw_options_2 if it is not the default
-                if raw_options_2 is not None:
-                    value, default = get_option(raw_options_2, setting)
-                    if not default:
-                        group_options[setting_name] = value
-        options[group_name] = group_options
-    # options["definitions"] = definitions
-    return options
+                elif setting_name in group_options:
+                    del group_options[setting_name]
+        normalized[group_name] = group_options
+    return Config(normalized, definitions)
 
 
-def parse_file(definitions: Definitions, filename: pathlib.Path) -> tuple[Values, bool]:
+def parse_file(definitions: Definitions, filename: pathlib.Path) -> tuple[Config[Values], bool]:
     """
     Helper function to read options from a json dictionary from a file
     Args:
@@ -205,16 +301,16 @@ def parse_file(definitions: Definitions, filename: pathlib.Path) -> tuple[Values
             success = False
     else:
         logger.info('No config file found')
-        success = False
+        success = True
 
-    return (normalize_config(options, definitions, file=True), success)
+    return (normalize_config(Config(options, definitions), file=True), success)
 
 
 def clean_config(
-    options: Values | argparse.Namespace, definitions: Definitions, file: bool = False, cmdline: bool = False,
+    config: Config[T], file: bool = False, cmdline: bool = False,
 ) -> Values:
     """
-    Normalizes options and then cleans up empty groups and removes 'definitions'
+    Normalizes options and then cleans up empty groups
     Args:
         options:
         file:
@@ -224,42 +320,62 @@ def clean_config(
 
     """
 
-    clean_options = normalize_config(options, definitions, file=file, cmdline=cmdline)
+    clean_options, definitions = normalize_config(config, file=file, cmdline=cmdline)
     for group in list(clean_options.keys()):
         if not clean_options[group]:
             del clean_options[group]
     return clean_options
 
 
-def defaults(definitions: Definitions) -> Values:
-    return normalize_config({}, definitions, file=True, cmdline=True)
+def defaults(definitions: Definitions) -> Config[Values]:
+    return normalize_config(Config(Namespace(), definitions), file=True, cmdline=True)
 
 
-def get_namespace(options: Values, definitions: Definitions, defaults: bool = True) -> argparse.Namespace:
+def get_namespace(config: Config[T], defaults: bool = True, persistent: bool = True) -> Config[Namespace]:
     """
-    Returns an argparse.Namespace object with options in the form "{group_name}_{setting_name}"
+    Returns an Namespace object with options in the form "{group_name}_{setting_name}"
     `options` should already be normalized.
     Throws an exception if the internal_name is duplicated
 
     Args:
         options: Normalized options to turn into a Namespace
         defaults: Include default values in the returned dict
+        persistent: Include unknown keys in persistent groups
     """
-    options = normalize_config(options, definitions, file=True, cmdline=True)
-    namespace = argparse.Namespace()
-    for group_name, group in definitions.items():
-        for setting_name, setting in group.items():
-            if hasattr(namespace, setting.internal_name):
-                raise Exception(f'Duplicate internal name: {setting.internal_name}')
-            value, default = get_option(options, setting)
 
-            if not default or default and defaults:
-                setattr(namespace, setting.internal_name, value)
-    return namespace
+    if isinstance(config.values, Namespace):
+        options, definitions = normalize_config(config)
+    else:
+        options, definitions = config
+    namespace = Namespace()
+    for group_name, group in definitions.items():
+        if group.persistent and persistent:
+            group_options = get_options(config, group_name)
+            for name, value in group_options.items():
+                if name in group.v:
+                    internal_name, default = group.v[name].internal_name, group.v[name].default == value
+                else:
+                    internal_name, default = f'{group_name}_' + sanitize_name(name), None
+
+                if hasattr(namespace, internal_name):
+                    raise Exception(f'Duplicate internal name: {internal_name}')
+
+                if not default or default and defaults:
+                    setattr(namespace, internal_name, value)
+
+        else:
+            for setting_name, setting in group.v.items():
+                if hasattr(namespace, setting.internal_name):
+                    raise Exception(f'Duplicate internal name: {setting.internal_name}')
+                value, default = get_option(options, setting)
+
+                if not default or default and defaults:
+                    setattr(namespace, setting.internal_name, value)
+    return Config(namespace, definitions)
 
 
 def save_file(
-    options: Values | argparse.Namespace, definitions: Definitions, filename: pathlib.Path,
+    config: Config[T], filename: pathlib.Path,
 ) -> bool:
     """
     Helper function to save options from a json dictionary to a file
@@ -267,14 +383,14 @@ def save_file(
         options: The options to save to a json dictionary
         filename: A pathlib.Path object to save the json dictionary to
     """
-    file_options = clean_config(options, definitions, file=True)
+    file_options = clean_config(config, file=True)
     if not filename.exists():
         filename.parent.mkdir(exist_ok=True, parents=True)
         filename.touch()
 
     try:
         json_str = json.dumps(file_options, indent=2)
-        filename.write_text(json_str, encoding='utf-8')
+        filename.write_text(json_str + '\n', encoding='utf-8')
     except Exception:
         logger.exception('Failed to save config file: %s', filename)
         return False
@@ -288,7 +404,7 @@ def create_argparser(definitions: Definitions, description: str, epilog: str) ->
         description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter,
     )
     for group_name, group in definitions.items():
-        for setting_name, setting in group.items():
+        for setting_name, setting in group.v.items():
             if setting.cmdline:
                 argparse_args, argparse_kwargs = setting.to_argparse()
                 current_group: ArgParser = argparser
@@ -313,8 +429,8 @@ def parse_cmdline(
     description: str,
     epilog: str,
     args: list[str] | None = None,
-    namespace: argparse.Namespace | None = None,
-) -> Values:
+    config: Namespace | Config[T] | None = None,
+) -> Config[Values]:
     """
     Creates an `argparse.ArgumentParser` from cmdline settings in `self.definitions`.
     `args` and `namespace` are passed to `argparse.ArgumentParser.parse_args`
@@ -323,10 +439,18 @@ def parse_cmdline(
         args: Passed to argparse.ArgumentParser.parse
         namespace: Passed to argparse.ArgumentParser.parse
     """
+    namespace = None
+    if isinstance(config, Config):
+        if isinstance(config.values, Namespace):
+            namespace = config.values
+        else:
+            namespace = get_namespace(config, defaults=False)[0]
+    else:
+        namespace = config
     argparser = create_argparser(definitions, description, epilog)
     ns = argparser.parse_args(args, namespace=namespace)
 
-    return normalize_config(ns, definitions=definitions, cmdline=True, file=True)
+    return normalize_config(Config(ns, definitions), cmdline=True, file=True)
 
 
 def parse_config(
@@ -335,26 +459,29 @@ def parse_config(
     epilog: str,
     config_path: pathlib.Path,
     args: list[str] | None = None,
-) -> tuple[Values, bool]:
+) -> tuple[Config[Values], bool]:
     file_options, success = parse_file(definitions, config_path)
     cmdline_options = parse_cmdline(
-        definitions, description, epilog, args, get_namespace(file_options, definitions, defaults=False),
+        definitions, description, epilog, args, get_namespace(file_options, defaults=False),
     )
 
-    final_options = normalize_config(cmdline_options, definitions=definitions, file=True, cmdline=True)
+    final_options = normalize_config(cmdline_options, file=True, cmdline=True)
     return (final_options, success)
 
 
 class Manager:
     """docstring for Manager"""
 
-    def __init__(self, description: str = '', epilog: str = '', definitions: Definitions | None = None):
+    def __init__(self, description: str = '', epilog: str = '', definitions: Definitions | Config[T] | None = None):
         # This one is never used, it just makes MyPy happy
         self.argparser = argparse.ArgumentParser(description=description, epilog=epilog)
         self.description = description
         self.epilog = epilog
 
-        self.definitions: Definitions = defaultdict(lambda: dict(), definitions or {})
+        if isinstance(definitions, Config):
+            self.definitions = definitions.definitions
+        else:
+            self.definitions = defaultdict(lambda: Group(False, {}), definitions or {})
 
         self.exclusive_group = False
         self.current_group_name = ''
@@ -364,21 +491,37 @@ class Manager:
 
     def add_setting(self, *args: Any, **kwargs: Any) -> None:
         """Takes passes all arguments through to `Setting`, `group` and `exclusive` are already set"""
-        setting = Setting(*args, group=self.current_group_name, exclusive=self.exclusive_group, **kwargs)
-        self.definitions[self.current_group_name][setting.dest] = setting
+        setting = Setting(*args, **kwargs, group=self.current_group_name, exclusive=self.exclusive_group)
+        self.definitions[self.current_group_name].v[setting.dest] = setting
 
-    def add_group(self, name: str, add_settings: Callable[[Manager], None], exclusive_group: bool = False) -> None:
+    def add_group(self, name: str, group: Callable[[Manager], None], exclusive_group: bool = False) -> None:
         """
         The primary way to add define options on this class
 
         Args:
             name: The name of the group to define
-            add_settings: A function that registers individual options using :meth:`add_setting`
+            group: A function that registers individual options using :meth:`add_setting`
             exclusive_group: If this group is an argparse exclusive group
         """
         self.current_group_name = name
         self.exclusive_group = exclusive_group
-        add_settings(self)
+        group(self)
+        self.current_group_name = ''
+        self.exclusive_group = False
+
+    def add_persistent_group(self, name: str, group: Callable[[Manager], None], exclusive_group: bool = False) -> None:
+        """
+        The primary way to add define options on this class
+
+        Args:
+            name: The name of the group to define
+            group: A function that registers individual options using :meth:`add_setting`
+            exclusive_group: If this group is an argparse exclusive group
+        """
+        self.current_group_name = name
+        self.exclusive_group = exclusive_group
+        self.definitions[self.current_group_name] = Group(True, {})
+        group(self)
         self.current_group_name = ''
         self.exclusive_group = False
 
@@ -387,49 +530,64 @@ class Manager:
         self.argparser.exit(*args, **kwargs)
         raise SystemExit(99)
 
-    def defaults(self) -> Values:
+    def defaults(self) -> Config[Values]:
         return defaults(self.definitions)
 
     def clean_config(
-        self, options: Values | argparse.Namespace, file: bool = False, cmdline: bool = False,
+        self, options: T | Config[T], file: bool = False, cmdline: bool = False,
     ) -> Values:
-        return clean_config(options=options, definitions=self.definitions, file=file, cmdline=cmdline)
+        if isinstance(options, Config):
+            config = options
+        else:
+            config = Config(options, self.definitions)
+        return clean_config(config, file=file, cmdline=cmdline)
 
     def normalize_config(
         self,
-        raw_options: Values | argparse.Namespace,
+        options: T | Config[T],
         file: bool = False,
         cmdline: bool = False,
         defaults: bool = True,
-        raw_options_2: Values | argparse.Namespace | None = None,
-    ) -> Config:
-        return Config(
-            normalize_config(
-                raw_options=raw_options,
-                definitions=self.definitions,
-                file=file,
-                cmdline=cmdline,
-                defaults=defaults,
-                raw_options_2=raw_options_2,
-            ),
-            self.definitions,
+    ) -> Config[Values]:
+        if isinstance(options, Config):
+            config = options
+        else:
+            config = Config(options, self.definitions)
+        return normalize_config(
+            config=config,
+            file=file,
+            cmdline=cmdline,
+            defaults=defaults,
         )
 
-    def get_namespace(self, options: Values, defaults: bool = True) -> argparse.Namespace:
-        return get_namespace(options=options, definitions=self.definitions, defaults=defaults)
+    @overload
+    def get_namespace(self, options: Values, defaults: bool = True) -> Namespace:
+        ...
 
-    def parse_file(self, filename: pathlib.Path) -> tuple[Values, bool]:
+    @overload
+    def get_namespace(self, options: Config[Values], defaults: bool = True) -> Config[Namespace]:
+        ...
+
+    def get_namespace(self, options: Values | Config[Values], defaults: bool = True) -> Config[Namespace] | Namespace:
+        if isinstance(options, Config):
+            self.definitions = options[1]
+            return get_namespace(options, defaults=defaults)
+        else:
+            return get_namespace(Config(options, self.definitions), defaults=defaults)
+
+    def parse_file(self, filename: pathlib.Path) -> tuple[Config[Values], bool]:
         return parse_file(filename=filename, definitions=self.definitions)
 
-    def save_file(self, options: Values | argparse.Namespace, filename: pathlib.Path) -> bool:
-        return save_file(options=options, definitions=self.definitions, filename=filename)
+    def save_file(self, options: T | Config[T], filename: pathlib.Path) -> bool:
+        if isinstance(options, Config):
+            return save_file(options, filename=filename)
+        return save_file(Config(options, self.definitions), filename=filename)
 
-    def parse_cmdline(self, args: list[str] | None = None, namespace: argparse.Namespace | None = None) -> Values:
+    def parse_cmdline(self, args: list[str] | None = None, namespace: ns[T] = None) -> Config[Values]:
         return parse_cmdline(self.definitions, self.description, self.epilog, args, namespace)
 
-    def parse_config(self, config_path: pathlib.Path, args: list[str] | None = None) -> tuple[Config, bool]:
-        values, success = parse_config(self.definitions, self.description, self.epilog, config_path, args)
-        return (Config(values, self.definitions), success)
+    def parse_config(self, config_path: pathlib.Path, args: list[str] | None = None) -> tuple[Config[Values], bool]:
+        return parse_config(self.definitions, self.description, self.epilog, config_path, args)
 
 
 def example(manager: Manager) -> None:
@@ -446,27 +604,40 @@ def example(manager: Manager) -> None:
     manager.add_setting(
         '--verbose', '-v',
         default=False,
-        action=argparse.BooleanOptionalAction,
+        action=BooleanOptionalAction,  # Added in Python 3.9
     )
 
 
-if __name__ == '__main__':
+def persistent(manager: Manager) -> None:
+    manager.add_setting(
+        '--test', '-t',
+        default=False,
+        action=BooleanOptionalAction,  # Added in Python 3.9
+    )
+
+
+def _main(args: list[str] | None = None) -> None:
     settings_path = pathlib.Path('./settings.json')
     manager = Manager(description='This is an example', epilog='goodbye!')
 
     manager.add_group('example', example)
+    manager.add_persistent_group('persistent', persistent)
 
     file_config, success = manager.parse_file(settings_path)
     file_namespace = manager.get_namespace(file_config)
 
-    merged_config = manager.parse_cmdline(namespace=file_namespace)
+    merged_config = manager.parse_cmdline(args=args, namespace=file_namespace)
     merged_namespace = manager.get_namespace(merged_config)
 
-    print(f'Hello {merged_config["example"]["hello"]}')  # noqa: T201
-    if merged_namespace.example_save:
+    print(f'Hello {merged_config.values["example"]["hello"]}')  # noqa: T201
+    if merged_namespace.values.example_save:
         if manager.save_file(merged_config, settings_path):
             print(f'Successfully saved settings to {settings_path}')  # noqa: T201
         else:
             print(f'Failed saving settings to a {settings_path}')  # noqa: T201
-    if merged_namespace.example_verbose:
-        print(f'{merged_namespace.example_verbose=}')  # noqa: T201
+    if merged_namespace.values.example_verbose:
+        print(f'{merged_namespace.values.example_verbose=}')  # noqa: T201
+
+
+if __name__ == '__main__':
+    _main()

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = settngs
-version = 0.2.0
+version = 0.5.0
 description = A library for managing settings
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -18,6 +18,8 @@ classifiers =
 
 [options]
 py_modules = settngs
+install_requires =
+    typing-extensions;python_version < '3.11'
 python_requires = >=3.8
 
 [options.packages.find]

testing/settngs.py

@@ -1,7 +1,107 @@
 from __future__ import annotations
 
 import pytest
+example: list[tuple[list[str], str, str]] = [
+    (
+        [],
+        'Hello world\n',
+        '',
+    ),
+    (
+        ['--hello', 'lordwelch'],
+        'Hello lordwelch\n',
+        '',
+    ),
+    (
+        ['--hello', 'lordwelch', '-s'],
+        'Hello lordwelch\nSuccessfully saved settings to settings.json\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
+    ),
+    (
+        [],
+        'Hello lordwelch\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
+    ),
+    (
+        ['-v'],
+        'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
+    ),
+    (
+        ['-v', '-s'],
+        'Hello lordwelch\nSuccessfully saved settings to settings.json\nmerged_namespace.values.example_verbose=True\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
+    ),
+    (
+        [],
+        'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
+    ),
+    (
+        ['manual settings.json'],
+        'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n',
+    ),
+    (
+        ['--no-verbose', '-t'],
+        'Hello lordwelch\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n',
+    ),
+    (
+        ['--no-verbose', '-s', '-t'],
+        'Hello lordwelch\nSuccessfully saved settings to settings.json\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  },\n  "persistent": {\n    "test": true,\n    "hello": "world"\n  }\n}\n',
+    ),
+    (
+        ['--hello', 'world', '--no-verbose', '--no-test', '-s'],
+        'Hello world\nSuccessfully saved settings to settings.json\n',
+        '{\n  "example": {\n    "hello": "world",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n',
+    ),
+    (
+        [],
+        'Hello world\n',
+        '{\n  "example": {\n    "hello": "world",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n',
+    ),
+]
 success = [
+    (
+        (
+            ('--test-setting',),
+            dict(
+                group='tst',
+            ),
+        ),  # Equivalent to Setting("--test-setting", group="tst")
+        {
+            'action': None,
+            'choices': None,
+            'cmdline': True,
+            'const': None,
+            'default': None,
+            'dest': 'test_setting',  # dest is calculated by Setting and is not used by argparse
+            'exclusive': False,
+            'file': True,
+            'group': 'tst',
+            'help': None,
+            'internal_name': 'tst_test_setting',  # Should almost always be "{group}_{dest}"
+            'metavar': 'TEST_SETTING',  # Set manually so argparse doesn't use TST_TEST
+            'nargs': None,
+            'required': None,
+            'type': None,
+            'argparse_args': ('--test-setting',),  # *args actually sent to argparse
+            'argparse_kwargs': {
+                'action': None,
+                'choices': None,
+                'const': None,
+                'default': None,
+                'dest': 'tst_test_setting',
+                'help': None,
+                'metavar': 'TEST_SETTING',
+                'nargs': None,
+                'required': None,
+                'type': None,
+            },  # Non-None **kwargs sent to argparse
+        },
+    ),
     (
         (
             ('--test',),
@@ -9,7 +109,7 @@ success = [
                 group='tst',
                 dest='testing',
             ),
-        ),  # Equivalent to Setting("--test", group="tst")
+        ),  # Equivalent to Setting("--test", group="tst", dest="testing")
         {
             'action': None,
             'choices': None,

tests/settngs_test.py

@@ -6,6 +6,8 @@ import json
 import pytest
 
 import settngs
+from settngs import Group
+from testing.settngs import example
 from testing.settngs import failure
 from testing.settngs import success
 
@@ -22,6 +24,19 @@ def test_settngs_manager():
     assert manager is not None and defaults is not None
 
 
+def test_settngs_manager_config():
+    manager = settngs.Manager(
+        definitions=settngs.Config[settngs.Namespace](
+            settngs.Namespace(),
+            {'tst': Group(False, {'test': settngs.Setting('--test', default='hello', group='tst', exclusive=False)})},
+        ),
+    )
+
+    defaults = manager.defaults()
+    assert manager is not None and defaults is not None
+    assert defaults.values['tst']['test'] == 'hello'
+
+
 @pytest.mark.parametrize('arguments, expected', success)
 def test_setting_success(arguments, expected):
     assert vars(settngs.Setting(*arguments[0], **arguments[1])) == expected
@@ -39,25 +54,31 @@ def test_add_setting(settngs_manager):
 
 def test_get_defaults(settngs_manager):
     settngs_manager.add_setting('--test', default='hello')
-    defaults = settngs_manager.defaults()
+    defaults, _ = settngs_manager.defaults()
     assert defaults['']['test'] == 'hello'
 
 
 def test_get_namespace(settngs_manager):
     settngs_manager.add_setting('--test', default='hello')
-    defaults = settngs_manager.get_namespace(settngs_manager.defaults())
+    defaults, _ = settngs_manager.get_namespace(settngs_manager.defaults())
+    assert defaults.test == 'hello'
+
+
+def test_get_namespace_with_namespace(settngs_manager):
+    settngs_manager.add_setting('--test', default='hello')
+    defaults, _ = settngs_manager.get_namespace(argparse.Namespace(test='hello'))
     assert defaults.test == 'hello'
 
 
 def test_get_defaults_group(settngs_manager):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
-    defaults = settngs_manager.defaults()
+    defaults, _ = settngs_manager.defaults()
     assert defaults['tst']['test'] == 'hello'
 
 
 def test_get_namespace_group(settngs_manager):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
-    defaults = settngs_manager.get_namespace(settngs_manager.defaults())
+    defaults, _ = settngs_manager.get_namespace(settngs_manager.defaults())
     assert defaults.tst_test == 'hello'
 
 
@@ -65,8 +86,8 @@ def test_cmdline_only(settngs_manager):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', file=False))
     settngs_manager.add_group('tst2', lambda parser: parser.add_setting('--test2', default='hello', cmdline=False))
 
-    file_normalized, _ = settngs_manager.normalize_config({}, file=True)
-    cmdline_normalized, _ = settngs_manager.normalize_config({}, cmdline=True)
+    file_normalized, _ = settngs_manager.normalize_config(settngs_manager.defaults(), file=True)
+    cmdline_normalized, _ = settngs_manager.normalize_config(settngs_manager.defaults(), cmdline=True)
 
     assert 'test' in cmdline_normalized['tst']
     assert 'test2' not in cmdline_normalized['tst2']
@@ -77,31 +98,39 @@ def test_cmdline_only(settngs_manager):
 
 def test_normalize(settngs_manager):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
+    settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
 
     defaults = settngs_manager.defaults()
-    defaults['test'] = 'fail'  # Not defined in settngs_manager
+    defaults.values['test'] = 'fail'  # Not defined in settngs_manager, should be removed
+    defaults.values['persistent']['hello'] = 'success'  # Not defined in settngs_manager, should stay
 
-    defaults_namespace = settngs_manager.get_namespace(defaults)
-    defaults_namespace.test = 'fail'
+    defaults_namespace = settngs_manager.get_namespace(settngs_manager.defaults())
+    defaults_namespace.values.test = 'fail'  # Not defined in settngs_manager, should be removed
+    defaults_namespace.values.persistent_hello = 'success'  # Not defined in settngs_manager, should stay
 
     normalized, _ = settngs_manager.normalize_config(defaults, file=True)
-    normalized_namespace = settngs_manager.get_namespace(settngs_manager.normalize_config(defaults, file=True)[0])
+    normalized_from_namespace = settngs_manager.normalize_config(defaults_namespace, file=True)
+    normalized_namespace, _ = settngs_manager.get_namespace(normalized_from_namespace)
 
     assert 'test' not in normalized
     assert 'tst' in normalized
     assert 'test' in normalized['tst']
     assert normalized['tst']['test'] == 'hello'
+    assert normalized['persistent']['hello'] == 'success'
 
     assert not hasattr(normalized_namespace, 'test')
     assert hasattr(normalized_namespace, 'tst_test')
     assert normalized_namespace.tst_test == 'hello'
+    assert normalized_namespace.persistent_hello == 'success'
 
 
 def test_clean_config(settngs_manager):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
     settngs_manager.add_group('tst2', lambda parser: parser.add_setting('--test2', default='hello', file=False))
-    normalized = settngs_manager.defaults()
+    settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
+    normalized, _ = settngs_manager.defaults()
     normalized['tst']['test'] = 'success'
+    normalized['persistent']['hello'] = 'success'
     normalized['fail'] = 'fail'
 
     cleaned = settngs_manager.clean_config(normalized, file=True)
@@ -109,12 +138,24 @@ def test_clean_config(settngs_manager):
     assert 'fail' not in cleaned
     assert 'tst2' not in cleaned
     assert cleaned['tst']['test'] == 'success'
+    assert cleaned['persistent']['hello'] == 'success'
 
 
 def test_parse_cmdline(settngs_manager, tmp_path):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=True))
 
-    normalized = settngs_manager.parse_cmdline(['--test', 'success'])
+    normalized, _ = settngs_manager.parse_cmdline(['--test', 'success'])
+
+    assert 'test' in normalized['tst']
+    assert normalized['tst']['test'] == 'success'
+
+
+def test_parse_cmdline_with_namespace(settngs_manager, tmp_path):
+    settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=True))
+
+    normalized, _ = settngs_manager.parse_cmdline(
+        ['--test', 'success'], namespace=settngs.Config({'tst': {'test': 'fail'}}, settngs_manager.definitions),
+    )
 
     assert 'test' in normalized['tst']
     assert normalized['tst']['test'] == 'success'
@@ -122,14 +163,16 @@ def test_parse_cmdline(settngs_manager, tmp_path):
 
 def test_parse_file(settngs_manager, tmp_path):
     settngs_file = tmp_path / 'settngs.json'
-    settngs_file.write_text(json.dumps({'tst': {'test': 'success'}}))
+    settngs_file.write_text(json.dumps({'tst': {'test': 'success'}, 'persistent': {'hello': 'success'}}))
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
+    settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
 
     normalized, success = settngs_manager.parse_file(settngs_file)
 
     assert success
-    assert 'test' in normalized['tst']
-    assert normalized['tst']['test'] == 'success'
+    assert 'test' in normalized[0]['tst']
+    assert normalized[0]['tst']['test'] == 'success'
+    assert normalized[0]['persistent']['hello'] == 'success'
 
 
 def test_parse_non_existent_file(settngs_manager, tmp_path):
@@ -138,9 +181,9 @@ def test_parse_non_existent_file(settngs_manager, tmp_path):
 
     normalized, success = settngs_manager.parse_file(settngs_file)
 
-    assert not success
-    assert 'test' in normalized['tst']
-    assert normalized['tst']['test'] == 'hello'
+    assert success
+    assert 'test' in normalized[0]['tst']
+    assert normalized[0]['tst']['test'] == 'hello'
 
 
 def test_parse_corrupt_file(settngs_manager, tmp_path):
@@ -151,54 +194,44 @@ def test_parse_corrupt_file(settngs_manager, tmp_path):
     normalized, success = settngs_manager.parse_file(settngs_file)
 
     assert not success
-    assert 'test' in normalized['tst']
-    assert normalized['tst']['test'] == 'hello'
+    assert 'test' in normalized[0]['tst']
+    assert normalized[0]['tst']['test'] == 'hello'
 
 
 def test_save_file(settngs_manager, tmp_path):
     settngs_file = tmp_path / 'settngs.json'
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
-    normalized = settngs_manager.defaults()
+    settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
+    normalized, _ = settngs_manager.defaults()
     normalized['tst']['test'] = 'success'
+    normalized['persistent']['hello'] = 'success'
 
     success = settngs_manager.save_file(normalized, settngs_file)
-    normalized, success_r = settngs_manager.parse_file(settngs_file)
+    normalized_r, success_r = settngs_manager.parse_file(settngs_file)
 
     assert success and success_r
-    assert 'test' in normalized['tst']
-    assert normalized['tst']['test'] == 'success'
+    assert 'test' in normalized_r[0]['tst']
+    assert normalized_r[0]['tst']['test'] == 'success'
+    assert normalized_r[0]['persistent']['hello'] == 'success'
 
 
 def test_save_file_not_seriazable(settngs_manager, tmp_path):
     settngs_file = tmp_path / 'settngs.json'
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
-    normalized = settngs_manager.defaults()
+    normalized, _ = settngs_manager.defaults()
     normalized['tst']['test'] = {'fail'}  # Sets are not serializabl
 
     success = settngs_manager.save_file(normalized, settngs_file)
-    normalized, success_r = settngs_manager.parse_file(settngs_file)
+    normalized_r, success_r = settngs_manager.parse_file(settngs_file)
+    # normalized_r will be the default settings
 
-    assert not (success and success_r)
+    assert not success
+    assert not success_r
     assert 'test' in normalized['tst']
-    assert normalized['tst']['test'] == 'hello'
+    assert normalized['tst']['test'] == {'fail'}
 
-
-@pytest.mark.parametrize(
-    'raw, raw2, expected',
-    [
-        ({'tst': {'test': 'fail'}}, argparse.Namespace(tst_test='success'), 'success'),
-        # hello is default so is not used in raw_options_2
-        ({'tst': {'test': 'success'}}, argparse.Namespace(tst_test='hello'), 'success'),
-        (argparse.Namespace(tst_test='fail'), {'tst': {'test': 'success'}}, 'success'),
-        (argparse.Namespace(tst_test='success'), {'tst': {'test': 'hello'}}, 'success'),
-    ],
-)
-def test_normalize_merge(raw, raw2, expected, settngs_manager):
-    settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
-
-    normalized, _ = settngs_manager.normalize_config(raw, file=True, raw_options_2=raw2)
-
-    assert normalized['tst']['test'] == expected
+    assert 'test' in normalized_r[0]['tst']
+    assert normalized_r[0]['tst']['test'] == 'hello'
 
 
 def test_cli_set(settngs_manager, tmp_path):
@@ -251,3 +284,21 @@ def test_cli_explicit_default(settngs_manager, tmp_path):
     assert success
     assert 'test' in normalized['tst']
     assert normalized['tst']['test'] == 'success'
+
+
+def test_example(capsys, tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    settings_file = tmp_path / 'settings.json'
+    settings_file.touch()
+
+    i = 0
+    for args, expected_out, expected_file in example:
+        if args == ['manual settings.json']:
+            settings_file.unlink()
+            settings_file.write_text('{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n')
+        else:
+            settngs._main(args)
+            captured = capsys.readouterr()
+        assert captured.out == expected_out, f'{i}, {args}'
+        assert settings_file.read_text() == expected_file, f'{i}, {args}'
+        i += 1