Version Bump

Fix fstring
Add comments explaining execution of normalize_config with defaults arg
Fix not removing file or cmdline settings in persistent groups
Fix a bug in get_namespace when config is a namespace
Add additional tests

.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
     -   id: pyupgrade
         args: [--py38-plus]
 -   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v2.0.0
+    rev: v2.0.1
     hooks:
     -   id: autopep8
 -   repo: https://github.com/PyCQA/flake8
@@ -42,6 +42,6 @@ repos:
     -   id: flake8
         additional_dependencies: [flake8-encodings, flake8-warnings, flake8-builtins, flake8-length, flake8-print]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.0.1
     hooks:
     -   id: mypy

README.md

@@ -16,7 +16,7 @@ pip install settngs
 ```
 
 
-A trivial example is included at the bottom of settngs.py with the output below. For a more complete example see [ComicTagger].
+A trivial example is included at the bottom of settngs.py with the output below (using bash). For a more complete example see [ComicTagger].
 ```console
 $ python -m settngs
 Hello world
@@ -37,6 +37,18 @@ merged_namespace.values.example_verbose=True
 $ python -m settngs
 Hello lordwelch
 merged_namespace.values.example_verbose=True
+$ cat >settings.json << EOF
+{
+  "example": {
+    "hello": "lordwelch",
+    "verbose": true
+  },
+  "persistent": {
+    "test": false,
+    "hello": "world"
+  }
+}
+EOF
 $ python -m settngs --no-verbose
 Hello lordwelch
 $ python -m settngs --no-verbose -s
@@ -55,7 +67,9 @@ settngs.json at the end:
   "example": {
     "hello": "world",
     "verbose": false
-  }
+  },
+  "persistent": false,
+  "hello": "world"
 }
 ```
 

settngs.py

@@ -4,6 +4,7 @@ import argparse
 import json
 import logging
 import pathlib
+import re
 import sys
 from argparse import Namespace
 from collections import defaultdict
@@ -24,7 +25,14 @@ if sys.version_info < (3, 11):  # pragma: no cover
 else:  # pragma: no cover
     from typing import NamedTuple
 
+
 if sys.version_info < (3, 9):  # pragma: no cover
+    def removeprefix(self: str, prefix: str, /) -> str:
+        if self.startswith(prefix):
+            return self[len(prefix):]
+        else:
+            return self[:]
+
     class BooleanOptionalAction(argparse.Action):
         def __init__(
             self,
@@ -66,6 +74,7 @@ if sys.version_info < (3, 9):  # pragma: no cover
                 setattr(namespace, self.dest, not option_string.startswith('--no-'))
 else:  # pragma: no cover
     from argparse import BooleanOptionalAction
+    removeprefix = str.removeprefix
 
 
 class Setting:
@@ -84,11 +93,35 @@ class Setting:
         metavar: str | None = None,
         dest: str | None = None,
         # ComicTagger
+        display_name: str = '',
         cmdline: bool = True,
         file: bool = True,
         group: str = '',
         exclusive: bool = False,
     ):
+        """
+
+        Args:
+            *names:       Passed directly to argparse
+            action:       Passed directly to argparse
+            nargs:        Passed directly to argparse
+            const:        Passed directly to argparse
+            default:      Passed directly to argparse
+            type:         Passed directly to argparse
+            choices:      Passed directly to argparse
+            required:     Passed directly to argparse
+            help:         Passed directly to argparse
+            metavar:      Passed directly to argparse, defaults to `dest` uppercased
+            dest:         This is the name used to retrieve the value from a `Config` object as a dictionary
+            display_name: This is not used by settngs. This is a human-readable name to be used when generating a GUI.
+                          Defaults to `dest`.
+            cmdline:      If this setting can be set via the commandline
+            file:         If this setting can be set via a file
+            group:        The group this option is in.
+                          This is an internal argument and should only be set by settngs
+            exclusive:    If this setting is exclusive to other settings in this group.
+                          This is an internal argument and should only be set by settngs
+        """
         if not names:
             raise ValueError('names must be specified')
         # We prefix the destination name used by argparse so that there are no conflicts
@@ -121,6 +154,7 @@ class Setting:
         self.argparse_args = args
         self.group = group
         self.exclusive = exclusive
+        self.display_name = display_name or dest
 
         self.argparse_kwargs = {
             'action': action,
@@ -141,6 +175,11 @@ class Setting:
     def __repr__(self) -> str:  # pragma: no cover
         return self.__str__()
 
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Setting):
+            return NotImplemented
+        return self.__dict__ == other.__dict__
+
     def get_dest(self, prefix: str, names: Sequence[str], dest: str | None) -> tuple[str, str, bool]:
         dest_name = None
         flag = False
@@ -148,7 +187,7 @@ class Setting:
         for n in names:
             if n.startswith('--'):
                 flag = True
-                dest_name = n.lstrip('-').replace('-', '_')
+                dest_name = sanitize_name(n)
                 break
             if n.startswith('-'):
                 flag = True
@@ -157,6 +196,8 @@ class Setting:
             dest_name = names[0]
         if dest:
             dest_name = dest
+        if not dest_name.isidentifier():
+            raise Exception(f'Cannot use {dest_name} in a namespace')
 
         internal_name = f'{prefix}_{dest_name}'.lstrip('_')
         return internal_name, dest_name, flag
@@ -168,8 +209,13 @@ class Setting:
         return self.argparse_args, self.filter_argparse_kwargs()
 
 
+class Group(NamedTuple):
+    persistent: bool
+    v: dict[str, Setting]
+
+
 Values = Dict[str, Dict[str, Any]]
-Definitions = Dict[str, Dict[str, Setting]]
+Definitions = Dict[str, Group]
 
 T = TypeVar('T', Values, Namespace)
 
@@ -184,6 +230,10 @@ if TYPE_CHECKING:
     ns = Namespace | Config[T] | None
 
 
+def sanitize_name(name: str) -> str:
+    return re.sub('[' + re.escape(' -_,.!@#$%^&*(){}[]\',."<>;:') + ']+', '_', name).strip('_')
+
+
 def get_option(options: Values | Namespace, setting: Setting) -> tuple[Any, bool]:
     """
     Helper function to retrieve the value for a setting and if the value is the default value
@@ -199,11 +249,36 @@ def get_option(options: Values | Namespace, setting: Setting) -> tuple[Any, bool
     return value, value == setting.default
 
 
+def get_options(options: Config[T], group: str) -> dict[str, Any]:
+    """
+    Helper function to retrieve all of the values for a group. Only to be used on persistent groups.
+
+    Args:
+        options: Dictionary or namespace of options
+        group: The name of the group to retrieve
+    """
+    if isinstance(options[0], dict):
+        values = options[0].get(group, {}).copy()
+    else:
+        internal_names = {x.internal_name: x for x in options[1][group].v.values()}
+        values = {}
+        v = vars(options[0])
+        for name, value in v.items():
+            if name.startswith(f'{group}_'):
+                if name in internal_names:
+                    values[internal_names[name].dest] = value
+                else:
+                    values[removeprefix(name, f'{group}_')] = value
+
+    return values
+
+
 def normalize_config(
     config: Config[T],
     file: bool = False,
     cmdline: bool = False,
     defaults: bool = True,
+    persistent: bool = True,
 ) -> Config[Values]:
     """
     Creates an `OptionValues` dictionary with setting definitions taken from `self.definitions`
@@ -216,19 +291,28 @@ def normalize_config(
         file: Include file options
         cmdline: Include cmdline options
         defaults: Include default values in the returned dict
-        raw_options_2: If set, merges non-default values into the returned dict
+        persistent: Include unknown keys in persistent groups
     """
 
     normalized: Values = {}
     options, definitions = config
     for group_name, group in definitions.items():
         group_options = {}
-        for setting_name, setting in group.items():
+        if group.persistent and persistent:
+            group_options = get_options(config, group_name)
+        for setting_name, setting in group.v.items():
             if (setting.cmdline and cmdline) or (setting.file and file):
                 # Ensures the option exists with the default if not already set
                 value, default = get_option(options, setting)
-                if not default or default and defaults:
+                if not default or (default and defaults):
+                    # User has set a custom value or has requested the default value
                     group_options[setting_name] = value
+                elif setting_name in group_options:
+                    # defaults have been requested to be removed
+                    del group_options[setting_name]
+            elif setting_name in group_options:
+                # Setting type (file or cmdline) has not been requested and should be removed for persistent groups
+                del group_options[setting_name]
         normalized[group_name] = group_options
     return Config(normalized, definitions)
 
@@ -261,7 +345,7 @@ def clean_config(
     config: Config[T], file: bool = False, cmdline: bool = False,
 ) -> Values:
     """
-    Normalizes options and then cleans up empty groups and removes 'definitions'
+    Normalizes options and then cleans up empty groups
     Args:
         options:
         file:
@@ -282,7 +366,7 @@ def defaults(definitions: Definitions) -> Config[Values]:
     return normalize_config(Config(Namespace(), definitions), file=True, cmdline=True)
 
 
-def get_namespace(config: Config[T], defaults: bool = True) -> Config[Namespace]:
+def get_namespace(config: Config[T], defaults: bool = True, persistent: bool = True) -> Config[Namespace]:
     """
     Returns an Namespace object with options in the form "{group_name}_{setting_name}"
     `options` should already be normalized.
@@ -291,21 +375,37 @@ def get_namespace(config: Config[T], defaults: bool = True) -> Config[Namespace]
     Args:
         options: Normalized options to turn into a Namespace
         defaults: Include default values in the returned dict
+        persistent: Include unknown keys in persistent groups
     """
 
     if isinstance(config.values, Namespace):
-        options, definitions = normalize_config(config)
+        options, definitions = normalize_config(config, defaults=defaults, persistent=persistent)
     else:
         options, definitions = config
     namespace = Namespace()
     for group_name, group in definitions.items():
-        for setting_name, setting in group.items():
-            if hasattr(namespace, setting.internal_name):
-                raise Exception(f'Duplicate internal name: {setting.internal_name}')
-            value, default = get_option(options, setting)
+        if group.persistent and persistent:
+            group_options = get_options(config, group_name)
+            for name, value in group_options.items():
+                if name in group.v:
+                    internal_name, default = group.v[name].internal_name, group.v[name].default == value
+                else:
+                    internal_name, default = f'{group_name}_' + sanitize_name(name), None
 
-            if not default or default and defaults:
-                setattr(namespace, setting.internal_name, value)
+                if hasattr(namespace, internal_name):
+                    raise Exception(f'Duplicate internal name: {internal_name}')
+
+                if not default or default and defaults:
+                    setattr(namespace, internal_name, value)
+
+        else:
+            for setting_name, setting in group.v.items():
+                if hasattr(namespace, setting.internal_name):
+                    raise Exception(f'Duplicate internal name: {setting.internal_name}')
+                value, default = get_option(options, setting)
+
+                if not default or default and defaults:
+                    setattr(namespace, setting.internal_name, value)
     return Config(namespace, definitions)
 
 
@@ -339,7 +439,7 @@ def create_argparser(definitions: Definitions, description: str, epilog: str) ->
         description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter,
     )
     for group_name, group in definitions.items():
-        for setting_name, setting in group.items():
+        for setting_name, setting in group.v.items():
             if setting.cmdline:
                 argparse_args, argparse_kwargs = setting.to_argparse()
                 current_group: ArgParser = argparser
@@ -364,7 +464,7 @@ def parse_cmdline(
     description: str,
     epilog: str,
     args: list[str] | None = None,
-    config: Namespace | Config[T] | None = None,
+    config: ns[T] = None,
 ) -> Config[Values]:
     """
     Creates an `argparse.ArgumentParser` from cmdline settings in `self.definitions`.
@@ -416,7 +516,7 @@ class Manager:
         if isinstance(definitions, Config):
             self.definitions = definitions.definitions
         else:
-            self.definitions = defaultdict(lambda: dict(), definitions or {})
+            self.definitions = defaultdict(lambda: Group(False, {}), definitions or {})
 
         self.exclusive_group = False
         self.current_group_name = ''
@@ -427,20 +527,45 @@ class Manager:
     def add_setting(self, *args: Any, **kwargs: Any) -> None:
         """Takes passes all arguments through to `Setting`, `group` and `exclusive` are already set"""
         setting = Setting(*args, **kwargs, group=self.current_group_name, exclusive=self.exclusive_group)
-        self.definitions[self.current_group_name][setting.dest] = setting
+        self.definitions[self.current_group_name].v[setting.dest] = setting
 
-    def add_group(self, name: str, add_settings: Callable[[Manager], None], exclusive_group: bool = False) -> None:
+    def add_group(self, name: str, group: Callable[[Manager], None], exclusive_group: bool = False) -> None:
         """
-        The primary way to add define options on this class
+        The primary way to add define options on this class.
 
         Args:
             name: The name of the group to define
-            add_settings: A function that registers individual options using :meth:`add_setting`
+            group: A function that registers individual options using :meth:`add_setting`
             exclusive_group: If this group is an argparse exclusive group
         """
+        if self.current_group_name != '':
+            raise ValueError('Sub groups are not allowed')
         self.current_group_name = name
         self.exclusive_group = exclusive_group
-        add_settings(self)
+        group(self)
+        self.current_group_name = ''
+        self.exclusive_group = False
+
+    def add_persistent_group(self, name: str, group: Callable[[Manager], None], exclusive_group: bool = False) -> None:
+        """
+        The primary way to add define options on this class.
+        This group allows existing values to persist even if there is no corresponding setting defined for it.
+
+        Args:
+            name: The name of the group to define
+            group: A function that registers individual options using :meth:`add_setting`
+            exclusive_group: If this group is an argparse exclusive group
+        """
+        if self.current_group_name != '':
+            raise ValueError('Sub groups are not allowed')
+        self.current_group_name = name
+        self.exclusive_group = exclusive_group
+        if self.current_group_name in self.definitions:
+            if not self.definitions[self.current_group_name].persistent:
+                raise ValueError('Group already existis and is not persistent')
+        else:
+            self.definitions[self.current_group_name] = Group(True, {})
+        group(self)
         self.current_group_name = ''
         self.exclusive_group = False
 
@@ -527,11 +652,20 @@ def example(manager: Manager) -> None:
     )
 
 
+def persistent(manager: Manager) -> None:
+    manager.add_setting(
+        '--test', '-t',
+        default=False,
+        action=BooleanOptionalAction,  # Added in Python 3.9
+    )
+
+
 def _main(args: list[str] | None = None) -> None:
     settings_path = pathlib.Path('./settings.json')
     manager = Manager(description='This is an example', epilog='goodbye!')
 
     manager.add_group('example', example)
+    manager.add_persistent_group('persistent', persistent)
 
     file_config, success = manager.parse_file(settings_path)
     file_namespace = manager.get_namespace(file_config)

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = settngs
-version = 0.3.0
+version = 0.6.1
 description = A library for managing settings
 long_description = file: README.md
 long_description_content_type = text/markdown

testing/settngs.py

@@ -15,50 +15,94 @@ example: list[tuple[list[str], str, str]] = [
     (
         ['--hello', 'lordwelch', '-s'],
         'Hello lordwelch\nSuccessfully saved settings to settings.json\n',
-        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  }\n}\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
     ),
     (
         [],
         'Hello lordwelch\n',
-        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  }\n}\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
     ),
     (
         ['-v'],
         'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n',
-        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  }\n}\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
     ),
     (
         ['-v', '-s'],
         'Hello lordwelch\nSuccessfully saved settings to settings.json\nmerged_namespace.values.example_verbose=True\n',
-        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  }\n}\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
     ),
     (
         [],
         'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n',
-        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  }\n}\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
     ),
     (
-        ['--no-verbose'],
+        ['manual settings.json'],
+        'Hello lordwelch\nmerged_namespace.values.example_verbose=True\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n',
+    ),
+    (
+        ['--no-verbose', '-t'],
         'Hello lordwelch\n',
-        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  }\n}\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n',
     ),
     (
-        ['--no-verbose', '-s'],
+        ['--no-verbose', '-s', '-t'],
         'Hello lordwelch\nSuccessfully saved settings to settings.json\n',
-        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  }\n}\n',
+        '{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": false\n  },\n  "persistent": {\n    "test": true,\n    "hello": "world"\n  }\n}\n',
     ),
     (
-        ['--hello', 'world', '--no-verbose', '-s'],
+        ['--hello', 'world', '--no-verbose', '--no-test', '-s'],
         'Hello world\nSuccessfully saved settings to settings.json\n',
-        '{\n  "example": {\n    "hello": "world",\n    "verbose": false\n  }\n}\n',
+        '{\n  "example": {\n    "hello": "world",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n',
     ),
     (
         [],
         'Hello world\n',
-        '{\n  "example": {\n    "hello": "world",\n    "verbose": false\n  }\n}\n',
+        '{\n  "example": {\n    "hello": "world",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n',
     ),
 ]
 success = [
+    (
+        (
+            ('--test-setting',),
+            dict(
+                group='tst',
+            ),
+        ),  # Equivalent to Setting("--test-setting", group="tst")
+        {
+            'action': None,
+            'choices': None,
+            'cmdline': True,
+            'const': None,
+            'default': None,
+            'dest': 'test_setting',  # dest is calculated by Setting and is not used by argparse
+            'display_name': 'test_setting',  # defaults to dest
+            'exclusive': False,
+            'file': True,
+            'group': 'tst',
+            'help': None,
+            'internal_name': 'tst_test_setting',  # Should almost always be "{group}_{dest}"
+            'metavar': 'TEST_SETTING',  # Set manually so argparse doesn't use TST_TEST
+            'nargs': None,
+            'required': None,
+            'type': None,
+            'argparse_args': ('--test-setting',),  # *args actually sent to argparse
+            'argparse_kwargs': {
+                'action': None,
+                'choices': None,
+                'const': None,
+                'default': None,
+                'dest': 'tst_test_setting',
+                'help': None,
+                'metavar': 'TEST_SETTING',
+                'nargs': None,
+                'required': None,
+                'type': None,
+            },  # Non-None **kwargs sent to argparse
+        },
+    ),
     (
         (
             ('--test',),
@@ -66,7 +110,7 @@ success = [
                 group='tst',
                 dest='testing',
             ),
-        ),  # Equivalent to Setting("--test", group="tst")
+        ),  # Equivalent to Setting("--test", group="tst", dest="testing")
         {
             'action': None,
             'choices': None,
@@ -74,6 +118,7 @@ success = [
             'const': None,
             'default': None,
             'dest': 'testing',  # dest is calculated by Setting and is not used by argparse
+            'display_name': 'testing',  # defaults to dest
             'exclusive': False,
             'file': True,
             'group': 'tst',
@@ -112,6 +157,7 @@ success = [
             'const': None,
             'default': None,
             'dest': 'test',  # dest is calculated by Setting and is not used by argparse
+            'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
             'group': 'tst',
@@ -151,6 +197,7 @@ success = [
             'const': None,
             'default': None,
             'dest': 'test',  # dest is calculated by Setting and is not used by argparse
+            'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
             'group': 'tst',
@@ -189,6 +236,7 @@ success = [
             'const': None,
             'default': None,
             'dest': 'test',
+            'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
             'group': 'tst',
@@ -227,6 +275,7 @@ success = [
             'const': None,
             'default': None,
             'dest': 'test',
+            'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
             'group': 'tst',
@@ -263,6 +312,7 @@ success = [
             'const': None,
             'default': None,
             'dest': 'test',
+            'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
             'group': '',

tests/settngs_test.py

@@ -2,10 +2,12 @@ from __future__ import annotations
 
 import argparse
 import json
+from collections import defaultdict
 
 import pytest
 
 import settngs
+from settngs import Group
 from testing.settngs import example
 from testing.settngs import failure
 from testing.settngs import success
@@ -27,7 +29,7 @@ def test_settngs_manager_config():
     manager = settngs.Manager(
         definitions=settngs.Config[settngs.Namespace](
             settngs.Namespace(),
-            {'tst': {'test': settngs.Setting('--test', default='hello', group='tst', exclusive=False)}},
+            {'tst': Group(False, {'test': settngs.Setting('--test', default='hello', group='tst', exclusive=False)})},
         ),
     )
 
@@ -57,7 +59,7 @@ def test_get_defaults(settngs_manager):
     assert defaults['']['test'] == 'hello'
 
 
-def test_get_namespace(settngs_manager):
+def test_get_defaults_namespace(settngs_manager):
     settngs_manager.add_setting('--test', default='hello')
     defaults, _ = settngs_manager.get_namespace(settngs_manager.defaults())
     assert defaults.test == 'hello'
@@ -65,8 +67,8 @@ def test_get_namespace(settngs_manager):
 
 def test_get_namespace_with_namespace(settngs_manager):
     settngs_manager.add_setting('--test', default='hello')
-    defaults, _ = settngs_manager.get_namespace(argparse.Namespace(test='hello'))
-    assert defaults.test == 'hello'
+    defaults, _ = settngs_manager.get_namespace(argparse.Namespace(test='success'))
+    assert defaults.test == 'success'
 
 
 def test_get_defaults_group(settngs_manager):
@@ -95,14 +97,31 @@ def test_cmdline_only(settngs_manager):
     assert 'test2' in file_normalized['tst2']
 
 
+def test_cmdline_only_persistent_group(settngs_manager):
+    settngs_manager.add_persistent_group('tst', lambda parser: parser.add_setting('--test', default='hello', file=False))
+    settngs_manager.add_group('tst2', lambda parser: parser.add_setting('--test2', default='hello', cmdline=False))
+
+    file_normalized, _ = settngs_manager.normalize_config(settngs_manager.defaults(), file=True)
+    cmdline_normalized, _ = settngs_manager.normalize_config(settngs_manager.defaults(), cmdline=True)
+
+    assert 'test' in cmdline_normalized['tst']
+    assert 'test2' not in cmdline_normalized['tst2']
+
+    assert 'test' not in file_normalized['tst']
+    assert 'test2' in file_normalized['tst2']
+
+
 def test_normalize(settngs_manager):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
+    settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
 
     defaults = settngs_manager.defaults()
-    defaults.values['test'] = 'fail'  # Not defined in settngs_manager
+    defaults.values['test'] = 'fail'  # Not defined in settngs_manager, should be removed
+    defaults.values['persistent']['hello'] = 'success'  # Not defined in settngs_manager, should stay
 
     defaults_namespace = settngs_manager.get_namespace(settngs_manager.defaults())
-    defaults_namespace.values.test = 'fail'
+    defaults_namespace.values.test = 'fail'  # Not defined in settngs_manager, should be removed
+    defaults_namespace.values.persistent_hello = 'success'  # Not defined in settngs_manager, should stay
 
     normalized, _ = settngs_manager.normalize_config(defaults, file=True)
     normalized_from_namespace = settngs_manager.normalize_config(defaults_namespace, file=True)
@@ -112,17 +131,23 @@ def test_normalize(settngs_manager):
     assert 'tst' in normalized
     assert 'test' in normalized['tst']
     assert normalized['tst']['test'] == 'hello'
+    assert normalized['persistent']['hello'] == 'success'
+    assert normalized['persistent']['world'] == 'world'
 
     assert not hasattr(normalized_namespace, 'test')
     assert hasattr(normalized_namespace, 'tst_test')
     assert normalized_namespace.tst_test == 'hello'
+    assert normalized_namespace.persistent_hello == 'success'
+    assert normalized_namespace.persistent_world == 'world'
 
 
 def test_clean_config(settngs_manager):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
     settngs_manager.add_group('tst2', lambda parser: parser.add_setting('--test2', default='hello', file=False))
+    settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
     normalized, _ = settngs_manager.defaults()
     normalized['tst']['test'] = 'success'
+    normalized['persistent']['hello'] = 'success'
     normalized['fail'] = 'fail'
 
     cleaned = settngs_manager.clean_config(normalized, file=True)
@@ -130,9 +155,10 @@ def test_clean_config(settngs_manager):
     assert 'fail' not in cleaned
     assert 'tst2' not in cleaned
     assert cleaned['tst']['test'] == 'success'
+    assert cleaned['persistent']['hello'] == 'success'
 
 
-def test_parse_cmdline(settngs_manager, tmp_path):
+def test_parse_cmdline(settngs_manager):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=True))
 
     normalized, _ = settngs_manager.parse_cmdline(['--test', 'success'])
@@ -141,27 +167,39 @@ def test_parse_cmdline(settngs_manager, tmp_path):
     assert normalized['tst']['test'] == 'success'
 
 
-def test_parse_cmdline_with_namespace(settngs_manager, tmp_path):
+namespaces = (
+    lambda definitions: settngs.Config({'tst': {'test': 'fail', 'test2': 'success'}}, definitions),
+    lambda definitions: settngs.Config(argparse.Namespace(tst_test='fail', tst_test2='success'), definitions),
+    lambda definitions: argparse.Namespace(tst_test='fail', tst_test2='success'),
+)
+
+
+@pytest.mark.parametrize('ns', namespaces)
+def test_parse_cmdline_with_namespace(settngs_manager, ns):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=True))
+    settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test2', default='fail', cmdline=True))
 
     normalized, _ = settngs_manager.parse_cmdline(
-        ['--test', 'success'], namespace=settngs.Config({'tst': {'test': 'fail'}}, settngs_manager.definitions),
+        ['--test', 'success'], namespace=ns(settngs_manager.definitions),
     )
 
     assert 'test' in normalized['tst']
     assert normalized['tst']['test'] == 'success'
+    assert normalized['tst']['test2'] == 'success'
 
 
 def test_parse_file(settngs_manager, tmp_path):
     settngs_file = tmp_path / 'settngs.json'
-    settngs_file.write_text(json.dumps({'tst': {'test': 'success'}}))
+    settngs_file.write_text(json.dumps({'tst': {'test': 'success'}, 'persistent': {'hello': 'success'}}))
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
+    settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
 
     normalized, success = settngs_manager.parse_file(settngs_file)
 
     assert success
     assert 'test' in normalized[0]['tst']
     assert normalized[0]['tst']['test'] == 'success'
+    assert normalized[0]['persistent']['hello'] == 'success'
 
 
 def test_parse_non_existent_file(settngs_manager, tmp_path):
@@ -190,15 +228,18 @@ def test_parse_corrupt_file(settngs_manager, tmp_path):
 def test_save_file(settngs_manager, tmp_path):
     settngs_file = tmp_path / 'settngs.json'
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', cmdline=False))
+    settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
     normalized, _ = settngs_manager.defaults()
     normalized['tst']['test'] = 'success'
+    normalized['persistent']['hello'] = 'success'
 
     success = settngs_manager.save_file(normalized, settngs_file)
-    normalized, success_r = settngs_manager.parse_file(settngs_file)
+    normalized_r, success_r = settngs_manager.parse_file(settngs_file)
 
     assert success and success_r
-    assert 'test' in normalized[0]['tst']
-    assert normalized[0]['tst']['test'] == 'success'
+    assert 'test' in normalized_r[0]['tst']
+    assert normalized_r[0]['tst']['test'] == 'success'
+    assert normalized_r[0]['persistent']['hello'] == 'success'
 
 
 def test_save_file_not_seriazable(settngs_manager, tmp_path):
@@ -208,11 +249,16 @@ def test_save_file_not_seriazable(settngs_manager, tmp_path):
     normalized['tst']['test'] = {'fail'}  # Sets are not serializabl
 
     success = settngs_manager.save_file(normalized, settngs_file)
-    normalized, success_r = settngs_manager.parse_file(settngs_file)
+    normalized_r, success_r = settngs_manager.parse_file(settngs_file)
+    # normalized_r will be the default settings
 
-    assert not (success and success_r)
-    assert 'test' in normalized[0]['tst']
-    assert normalized[0]['tst']['test'] == 'hello'
+    assert not success
+    assert not success_r
+    assert 'test' in normalized['tst']
+    assert normalized['tst']['test'] == {'fail'}
+
+    assert 'test' in normalized_r[0]['tst']
+    assert normalized_r[0]['tst']['test'] == 'hello'
 
 
 def test_cli_set(settngs_manager, tmp_path):
@@ -267,13 +313,55 @@ def test_cli_explicit_default(settngs_manager, tmp_path):
     assert normalized['tst']['test'] == 'success'
 
 
+def test_adding_to_existing_group(settngs_manager, tmp_path):
+    def default_to_regular(d):
+        if isinstance(d, defaultdict):
+            d = {k: default_to_regular(v) for k, v in d.items()}
+        return d
+    settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='success'))
+    settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test2', default='success'))
+
+    def tst(parser):
+        parser.add_setting('--test', default='success')
+        parser.add_setting('--test2', default='success')
+
+    settngs_manager2 = settngs.Manager()
+    settngs_manager2.add_group('tst', tst)
+
+    assert default_to_regular(settngs_manager.definitions) == default_to_regular(settngs_manager2.definitions)
+
+
+def test_adding_to_existing_persistent_group(settngs_manager, tmp_path):
+    def default_to_regular(d):
+        if isinstance(d, defaultdict):
+            d = {k: default_to_regular(v) for k, v in d.items()}
+        return d
+    settngs_manager.add_persistent_group('tst', lambda parser: parser.add_setting('--test', default='success'))
+    settngs_manager.add_persistent_group('tst', lambda parser: parser.add_setting('--test2', default='success'))
+
+    def tst(parser):
+        parser.add_setting('--test', default='success')
+        parser.add_setting('--test2', default='success')
+
+    settngs_manager2 = settngs.Manager()
+    settngs_manager2.add_persistent_group('tst', tst)
+
+    assert default_to_regular(settngs_manager.definitions) == default_to_regular(settngs_manager2.definitions)
+
+
 def test_example(capsys, tmp_path, monkeypatch):
     monkeypatch.chdir(tmp_path)
     settings_file = tmp_path / 'settings.json'
     settings_file.touch()
 
+    i = 0
     for args, expected_out, expected_file in example:
-        settngs._main(args)
-        captured = capsys.readouterr()
-        assert captured.out == expected_out, args
-        assert settings_file.read_text() == expected_file, args
+        if args == ['manual settings.json']:
+            settings_file.unlink()
+            settings_file.write_text('{\n  "example": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n')
+        else:
+            settngs._main(args)
+            captured = capsys.readouterr()
+        assert captured.out == expected_out, f'{i}, {args}'
+        assert settings_file.read_text() == expected_file, f'{i}, {args}'
+        i += 1