Fix normalization of settings using a custom dest

Improve formatting of namespace
Allow _guess_type to return any string
Add tests

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -10,11 +10,11 @@ repos:
     -   id: name-tests-test
     -   id: requirements-txt-fixer
 -   repo: https://github.com/asottile/setup-cfg-fmt
-    rev: v2.4.0
+    rev: v2.5.0
     hooks:
     -   id: setup-cfg-fmt
 -   repo: https://github.com/asottile/reorder-python-imports
-    rev: v3.10.0
+    rev: v3.12.0
     hooks:
     -   id: reorder-python-imports
         args: [--py38-plus, --add-import, 'from __future__ import annotations']
@@ -28,7 +28,7 @@ repos:
     hooks:
     -   id: dead
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.15.0
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -42,6 +42,6 @@ repos:
     -   id: flake8
         additional_dependencies: [flake8-encodings, flake8-warnings, flake8-builtins, flake8-length, flake8-print]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.7.0
     hooks:
     -   id: mypy

README.md

@@ -29,14 +29,14 @@ $ python -m settngs
 Hello lordwelch
 $ python -m settngs -v
 Hello lordwelch
-merged_namespace.values.example_verbose=True
+merged_namespace.values.Example_Group__verbose=True
 $ python -m settngs -v -s
 Hello lordwelch
 Successfully saved settings to settings.json
-merged_namespace.values.example_verbose=True
+merged_namespace.values.Example_Group__verbose=True
 $ python -m settngs
 Hello lordwelch
-merged_namespace.values.example_verbose=True
+merged_namespace.values.Example_Group__verbose=True
 $ cat >settings.json << EOF
 {
   "example": {
@@ -64,7 +64,7 @@ Hello world
 settngs.json at the end:
 ```json
 {
-  "example": {
+  "Example Group": {
     "hello": "world",
     "verbose": false
   },

settngs/__init__.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import copy
 import json
 import logging
 import pathlib
@@ -14,7 +15,6 @@ from typing import Any
 from typing import Callable
 from typing import Dict
 from typing import Generic
-from typing import Literal
 from typing import NoReturn
 from typing import TYPE_CHECKING
 from typing import TypeVar
@@ -107,6 +107,9 @@ class Setting:
         exclusive: bool = False,
     ):
         """
+        Attributes:
+        setting_name:     This is the name used to retrieve this Setting object from a `Config` Definitions dictionary.
+                          This only differs from dest when a custom dest is given
 
         Args:
             *names:       Passed directly to argparse
@@ -119,7 +122,8 @@ class Setting:
             required:     Passed directly to argparse
             help:         Passed directly to argparse
             metavar:      Passed directly to argparse, defaults to `dest` upper-cased
-            dest:         This is the name used to retrieve the value from a `Config` object as a dictionary
+            dest:         This is the name used to retrieve the value from a `Config` object as a dictionary.
+                          Default to `setting_name`.
             display_name: This is not used by settngs. This is a human-readable name to be used when generating a GUI.
                           Defaults to `dest`.
             cmdline:      If this setting can be set via the commandline
@@ -133,7 +137,7 @@ class Setting:
             raise ValueError('names must be specified')
         # We prefix the destination name used by argparse so that there are no conflicts
         # Argument names will still cause an exception if there is a conflict e.g. if '-f' is defined twice
-        self.internal_name, dest, flag = self.get_dest(group, names, dest)
+        self.internal_name, setting_name, dest, self.flag = self.get_dest(group, names, dest)
         args: Sequence[str] = names
 
         # We then also set the metavar so that '--config' in the group runtime shows as 'CONFIG' instead of 'RUNTIME_CONFIG'
@@ -142,7 +146,7 @@ class Setting:
 
         # If we are not a flag, no '--' or '-' in front
         # we use internal_name as argparse sets dest to args[0]
-        if not flag:
+        if not self.flag:
             args = tuple((self.internal_name, *names[1:]))
 
         self.action = action
@@ -155,6 +159,7 @@ class Setting:
         self.help = help
         self.metavar = metavar
         self.dest = dest
+        self.setting_name = setting_name
         self.cmdline = cmdline
         self.file = file
         self.argparse_args = args
@@ -172,7 +177,7 @@ class Setting:
             'required': required,
             'help': help,
             'metavar': metavar,
-            'dest': self.internal_name if flag else None,
+            'dest': self.internal_name if self.flag else None,
         }
 
     def __str__(self) -> str:  # pragma: no cover
@@ -186,7 +191,7 @@ class Setting:
             return NotImplemented
         return self.__dict__ == other.__dict__
 
-    def _guess_type(self) -> type | Literal['Any'] | None:
+    def _guess_type(self) -> type | str | None:
         if self.type is None and self.action is None:
             if self.cmdline:
                 if self.nargs in ('+', '*') or isinstance(self.nargs, int) and self.nargs > 1:
@@ -202,8 +207,9 @@ class Setting:
 
         if self.type is not None:
             type_hints = typing.get_type_hints(self.type)
-            if 'return' in type_hints and isinstance(type_hints['return'], type):
-                return type_hints['return']
+            if 'return' in type_hints:
+                t: type | str = type_hints['return']
+                return t
             if self.default is not None:
                 return type(self.default)
             return 'Any'
@@ -227,28 +233,30 @@ class Setting:
             return None
         return 'Any'
 
-    def get_dest(self, prefix: str, names: Sequence[str], dest: str | None) -> tuple[str, str, bool]:
-        dest_name = None
+    def get_dest(self, prefix: str, names: Sequence[str], dest: str | None) -> tuple[str, str, str, bool]:
+        setting_name = None
         flag = False
 
         prefix = sanitize_name(prefix)
         for n in names:
             if n.startswith('--'):
                 flag = True
-                dest_name = sanitize_name(n)
+                setting_name = sanitize_name(n)
                 break
             if n.startswith('-'):
                 flag = True
 
-        if dest_name is None:
-            dest_name = names[0]
+        if setting_name is None:
+            setting_name = names[0]
         if dest:
             dest_name = dest
+        else:
+            dest_name = setting_name
         if not dest_name.isidentifier():
             raise Exception(f'Cannot use {dest_name} in a namespace')
 
-        internal_name = f'{prefix}_{dest_name}'.lstrip('_')
-        return internal_name, dest_name, flag
+        internal_name = f'{prefix}__{dest_name}'.lstrip('_')
+        return internal_name, setting_name, dest_name, flag
 
     def filter_argparse_kwargs(self) -> dict[str, Any]:
         return {k: v for k, v in self.argparse_kwargs.items() if v is not None}
@@ -284,36 +292,58 @@ if TYPE_CHECKING:
 
 
 def generate_ns(definitions: Definitions) -> str:
-    imports = ['from __future__ import annotations', 'import typing', 'import settngs']
-    ns = 'class settngs_namespace(settngs.TypedNS):\n'
-    types = []
-    for group_name, group in definitions.items():
-        for setting_name, setting in group.v.items():
+    initial_imports = ['from __future__ import annotations', '', 'import settngs', '']
+    imports: Sequence[str] | set[str]
+    imports = set()
+
+    attributes = []
+    for group in definitions.values():
+        for setting in group.v.values():
             t = setting._guess_type()
             if t is None:
                 continue
+            # Default to any
             type_name = 'Any'
+
+            # Take a string as is
             if isinstance(t, str):
                 type_name = t
+            # Handle generic aliases eg dict[str, str] instead of dict
             elif isinstance(t, types_GenericAlias):
                 type_name = str(t)
+            # Handle standard type objects
             elif isinstance(t, type):
                 type_name = t.__name__
+                # Builtin types don't need an import
                 if t.__module__ != 'builtins':
-                    imports.append(f'import {t.__module__}')
+                    imports.add(f'import {t.__module__}')
+                    # Use the full imported name
                     type_name = t.__module__ + '.' + type_name
+
+            # Expand Any to typing.Any
             if type_name == 'Any':
                 type_name = 'typing.Any'
 
-            types.append(f'    {setting.internal_name}: {type_name}')
-        if types and types[-1] != '':
-            types.append('')
+            attributes.append(f'    {setting.internal_name}: {type_name}')
+        # Add a blank line between groups
+        if attributes and attributes[-1] != '':
+            attributes.append('')
 
-    if not types or all(x == '' for x in types):
+    ns = 'class settngs_namespace(settngs.TypedNS):\n'
+    # Add a '...' expression if there are no attributes
+    if not attributes or all(x == '' for x in attributes):
         ns += '    ...\n'
-        types = ['']
+        attributes = ['']
 
-    return '\n'.join(imports) + '\n\n' + ns + '\n'.join(types)
+    # Add the tying import before extra imports
+    if 'typing.' in '\n'.join(attributes):
+        initial_imports.append('import typing')
+
+    # Remove the possible duplicate typing import
+    imports = sorted(list(imports - {'import typing'}))
+
+    # Merge the imports the ns class definition and the attributes
+    return '\n'.join(initial_imports + imports) + '\n\n\n' + ns + '\n'.join(attributes)
 
 
 def sanitize_name(name: str) -> str:
@@ -354,11 +384,34 @@ def get_options(config: Config[T], group: str) -> dict[str, Any]:
                 if name in internal_names:
                     values[internal_names[name].dest] = value
                 else:
-                    values[removeprefix(name, f'{group}_')] = value
-
+                    values[removeprefix(name, f'{group}').lstrip('_')] = value
     return values
 
 
+def get_groups(values: Values | Namespace | TypedNS) -> list[str]:
+    if isinstance(values, dict):
+        return [x[0] for x in values.items() if isinstance(x[1], dict)]
+    if isinstance(values, Namespace):
+        groups = set()
+        for name in values.__dict__:
+            if '__' in name:
+                group, _, _ = name.partition('__')
+                groups.add(group.replace('_', ' '))
+            else:
+                groups.add('')
+        return list(groups)
+    return []
+
+
+def _get_internal_definitions(config: Config[T], persistent: bool) -> Definitions:
+    definitions = copy.deepcopy(dict(config.definitions))
+    if persistent:
+        for group_name in get_groups(config.values):
+            if group_name not in definitions:
+                definitions[group_name] = Group(True, {})
+    return defaultdict(lambda: Group(False, {}), definitions)
+
+
 def normalize_config(
     config: Config[T],
     file: bool = False,
@@ -376,33 +429,35 @@ def normalize_config(
         file: Include file options
         cmdline: Include cmdline options
         default: Include default values in the returned Config object
-        persistent: Include unknown keys in persistent groups
+        persistent: Include unknown keys in persistent groups and unknown groups
     """
 
     if not file and not cmdline:
         raise ValueError('Invalid parameters: you must set either file or cmdline to True')
 
     normalized: Values = {}
-    options, definitions = config
+    options = config.values
+    definitions = _get_internal_definitions(config=config, persistent=persistent)
     for group_name, group in definitions.items():
         group_options = {}
         if group.persistent and persistent:
-            group_options = get_options(config, group_name)
+            group_options = get_options(Config(options, definitions), group_name)
         for setting_name, setting in group.v.items():
             if (setting.cmdline and cmdline) or (setting.file and file):
                 # Ensures the option exists with the default if not already set
                 value, is_default = get_option(options, setting)
                 if not is_default or default:
                     # User has set a custom value or has requested the default value
-                    group_options[setting_name] = value
-                elif setting_name in group_options:
+                    group_options[setting.dest] = value
+                elif setting.dest in group_options:
                     # default values have been requested to be removed
-                    del group_options[setting_name]
-            elif setting_name in group_options:
+                    del group_options[setting.dest]
+            elif setting.dest in group_options:
                 # Setting type (file or cmdline) has not been requested and should be removed for persistent groups
-                del group_options[setting_name]
+                del group_options[setting.dest]
         normalized[group_name] = group_options
-    return Config(normalized, definitions)
+
+    return Config(normalized, config.definitions)
 
 
 def parse_file(definitions: Definitions, filename: pathlib.Path) -> tuple[Config[Values], bool]:
@@ -445,7 +500,7 @@ def clean_config(
         file: Include file options
         cmdline: Include cmdline options
         default: Include default values in the returned Config object
-        persistent: Include unknown keys in persistent groups
+        persistent: Include unknown keys in persistent groups and unknown groups
     """
 
     cleaned, _ = normalize_config(config, file=file, cmdline=cmdline, default=default, persistent=persistent)
@@ -471,24 +526,22 @@ def get_namespace(
         file: Include file options
         cmdline: Include cmdline options
         default: Include default values in the returned Config object
-        persistent: Include unknown keys in persistent groups
+        persistent: Include unknown keys in persistent groups and unknown groups
     """
-
     if not file and not cmdline:
         raise ValueError('Invalid parameters: you must set either file or cmdline to True')
 
     options: Values
-    definitions: Definitions
+    definitions = _get_internal_definitions(config=config, persistent=persistent)
     if isinstance(config.values, dict):
         options = config.values
-        definitions = config.definitions
     else:
         cfg = normalize_config(config, file=file, cmdline=cmdline, default=default, persistent=persistent)
-        options, definitions = cfg
+        options = cfg.values
     namespace = Namespace()
     for group_name, group in definitions.items():
 
-        group_options = get_options(config, group_name)
+        group_options = get_options(Config(options, definitions), group_name)
         if group.persistent and persistent:
             for name, value in group_options.items():
                 if name in group.v:
@@ -497,7 +550,7 @@ def get_namespace(
                     internal_name = group.v[name].internal_name
                 else:
                     setting_file = setting_cmdline = True
-                    internal_name, is_default = f'{group_name}_' + sanitize_name(name), None
+                    internal_name, is_default = f'{group_name}__' + sanitize_name(name), None
 
                 if ((setting_cmdline and cmdline) or (setting_file and file)) and (not is_default or default):
                     setattr(namespace, internal_name, value)
@@ -509,7 +562,7 @@ def get_namespace(
                 if not is_default or default:
                     # User has set a custom value or has requested the default value
                     setattr(namespace, setting.internal_name, value)
-    return Config(namespace, definitions)
+    return Config(namespace, config.definitions)
 
 
 def save_file(
@@ -558,8 +611,11 @@ def create_argparser(definitions: Definitions, description: str, epilog: str) ->
                         else:
                             groups[setting.group] = argparser.add_argument_group(setting.group)
 
-                    # hard coded exception for files
-                    if not (setting.group == 'runtime' and setting.nargs == '*'):
+                    # Hard coded exception for positional arguments
+                    # Ensures that the option shows at the top of the help output
+                    if 'runtime' in setting.group.casefold() and setting.nargs == '*' and not setting.flag:
+                        current_group = argparser
+                    else:
                         current_group = groups[setting.group]
                 current_group.add_argument(*argparse_args, **argparse_kwargs)
     return argparser
@@ -635,14 +691,20 @@ class Manager:
         self.description = description
         self.epilog = epilog
 
+        self.definitions: Definitions
         if isinstance(definitions, Config):
-            self.definitions = definitions.definitions
+            self.definitions = defaultdict(lambda: Group(False, {}), dict(definitions.definitions) or {})
         else:
-            self.definitions = defaultdict(lambda: Group(False, {}), definitions or {})
+            self.definitions = defaultdict(lambda: Group(False, {}), dict(definitions or {}))
 
         self.exclusive_group = False
         self.current_group_name = ''
 
+    def _get_config(self, c: T | Config[T]) -> Config[T]:
+        if not isinstance(c, Config):
+            return Config(c, self.definitions)
+        return c
+
     def generate_ns(self) -> str:
         return generate_ns(self.definitions)
 
@@ -651,8 +713,9 @@ class Manager:
 
     def add_setting(self, *args: Any, **kwargs: Any) -> None:
         """Passes all arguments through to `Setting`, `group` and `exclusive` are already set"""
+
         setting = Setting(*args, **kwargs, group=self.current_group_name, exclusive=self.exclusive_group)
-        self.definitions[self.current_group_name].v[setting.dest] = setting
+        self.definitions[self.current_group_name].v[setting.setting_name] = setting
 
     def add_group(self, name: str, group: Callable[[Manager], None], exclusive_group: bool = False) -> None:
         """
@@ -663,6 +726,7 @@ class Manager:
             group: A function that registers individual options using :meth:`add_setting`
             exclusive_group: If this group is an argparse exclusive group
         """
+
         if self.current_group_name != '':
             raise ValueError('Sub groups are not allowed')
         self.current_group_name = name
@@ -681,6 +745,7 @@ class Manager:
             group: A function that registers individual options using :meth:`add_setting`
             exclusive_group: If this group is an argparse exclusive group
         """
+
         if self.current_group_name != '':
             raise ValueError('Sub groups are not allowed')
         self.current_group_name = name
@@ -713,9 +778,7 @@ class Manager:
             cmdline: Include cmdline options
         """
 
-        if not isinstance(config, Config):
-            config = Config(config, self.definitions)
-        return clean_config(config, file=file, cmdline=cmdline)
+        return clean_config(self._get_config(config), file=file, cmdline=cmdline)
 
     def normalize_config(
         self,
@@ -735,13 +798,11 @@ class Manager:
             file: Include file options
             cmdline: Include cmdline options
             default: Include default values in the returned Config object
-            persistent: Include unknown keys in persistent groups
+            persistent: Include unknown keys in persistent groups and unknown groups
         """
 
-        if not isinstance(config, Config):
-            config = Config(config, self.definitions)
         return normalize_config(
-            config=config,
+            config=self._get_config(config),
             file=file,
             cmdline=cmdline,
             default=default,
@@ -750,7 +811,7 @@ class Manager:
 
     def get_namespace(
         self,
-        config: Values | Config[Values],
+        config: T | Config[T],
         file: bool = False,
         cmdline: bool = False,
         default: bool = True,
@@ -766,14 +827,12 @@ class Manager:
             file: Include file options
             cmdline: Include cmdline options
             default: Include default values in the returned Config object
-            persistent: Include unknown keys in persistent groups
+            persistent: Include unknown keys in persistent groups and unknown groups
         """
 
-        if isinstance(config, Config):
-            self.definitions = config[1]
-        else:
-            config = Config(config, self.definitions)
-        return get_namespace(config, file=file, cmdline=cmdline, default=default, persistent=persistent)
+        return get_namespace(
+            self._get_config(config), file=file, cmdline=cmdline, default=default, persistent=persistent,
+        )
 
     def parse_file(self, filename: pathlib.Path) -> tuple[Config[Values], bool]:
         """
@@ -784,6 +843,7 @@ class Manager:
         Args:
             filename: A pathlib.Path object to read a JSON dictionary from
         """
+
         return parse_file(filename=filename, definitions=self.definitions)
 
     def save_file(self, config: T | Config[T], filename: pathlib.Path) -> bool:
@@ -796,9 +856,8 @@ class Manager:
             config: The options to save to a json dictionary
             filename: A pathlib.Path object to save the json dictionary to
         """
-        if not isinstance(config, Config):
-            config = Config(config, self.definitions)
-        return save_file(config, filename=filename)
+
+        return save_file(self._get_config(config), filename=filename)
 
     def parse_cmdline(self, args: list[str] | None = None, config: ns[T] = None) -> Config[Values]:
         """
@@ -826,11 +885,12 @@ class Manager:
 
 __all__ = [
     'Setting',
+    'TypedNS',
     'Group',
     'Values',
     'Definitions',
     'Config',
-    'generate_settings',
+    'generate_ns',
     'sanitize_name',
     'get_option',
     'get_options',
@@ -887,13 +947,13 @@ def _main(args: list[str] | None = None) -> None:
     merged_namespace = manager.get_namespace(merged_config, file=True, cmdline=True)
 
     print(f'Hello {merged_config.values["Example Group"]["hello"]}')  # noqa: T201
-    if merged_namespace.values.Example_Group_save:
+    if merged_namespace.values.Example_Group__save:
         if manager.save_file(merged_config, settings_path):
             print(f'Successfully saved settings to {settings_path}')  # noqa: T201
-        else:
+        else:  # pragma: no cover
             print(f'Failed saving settings to a {settings_path}')  # noqa: T201
-    if merged_namespace.values.Example_Group_verbose:
-        print(f'{merged_namespace.values.Example_Group_verbose=}')  # noqa: T201
+    if merged_namespace.values.Example_Group__verbose:
+        print(f'{merged_namespace.values.Example_Group__verbose=}')  # noqa: T201
 
 
 if __name__ == '__main__':

setup.cfg

@@ -90,6 +90,7 @@ fail_under = 95
 [mypy]
 check_untyped_defs = true
 disallow_any_generics = true
+warn_return_any = true
 disallow_incomplete_defs = true
 disallow_untyped_defs = true
 no_implicit_optional = true
@@ -97,7 +98,9 @@ warn_redundant_casts = true
 warn_unused_ignores = true
 
 [mypy-testing.*]
+warn_return_any = false
 disallow_untyped_defs = false
 
 [mypy-tests.*]
+warn_return_any = false
 disallow_untyped_defs = false

testing/settngs.py

@@ -24,22 +24,22 @@ example: list[tuple[list[str], str, str]] = [
     ),
     (
         ['-v'],
-        'Hello lordwelch\nmerged_namespace.values.Example_Group_verbose=True\n',
+        'Hello lordwelch\nmerged_namespace.values.Example_Group__verbose=True\n',
         '{\n  "Example Group": {\n    "hello": "lordwelch",\n    "verbose": false\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
     ),
     (
         ['-v', '-s'],
-        'Hello lordwelch\nSuccessfully saved settings to settings.json\nmerged_namespace.values.Example_Group_verbose=True\n',
+        'Hello lordwelch\nSuccessfully saved settings to settings.json\nmerged_namespace.values.Example_Group__verbose=True\n',
         '{\n  "Example Group": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
     ),
     (
         [],
-        'Hello lordwelch\nmerged_namespace.values.Example_Group_verbose=True\n',
+        'Hello lordwelch\nmerged_namespace.values.Example_Group__verbose=True\n',
         '{\n  "Example Group": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false\n  }\n}\n',
     ),
     (
         ['manual settings.json'],
-        'Hello lordwelch\nmerged_namespace.values.Example_Group_verbose=True\n',
+        'Hello lordwelch\nmerged_namespace.values.Example_Group__verbose=True\n',
         '{\n  "Example Group": {\n    "hello": "lordwelch",\n    "verbose": true\n  },\n  "persistent": {\n    "test": false,\n    "hello": "world"\n  }\n}\n',
     ),
     (
@@ -77,13 +77,15 @@ success = [
             'cmdline': True,
             'const': None,
             'default': None,
+            'setting_name': 'test_setting',  # dest is calculated by Setting and is not used by argparse
             'dest': 'test_setting',  # dest is calculated by Setting and is not used by argparse
             'display_name': 'test_setting',  # defaults to dest
             'exclusive': False,
             'file': True,
+            'flag': True,
             'group': 'tst',
             'help': None,
-            'internal_name': 'tst_test_setting',  # Should almost always be "{group}_{dest}"
+            'internal_name': 'tst__test_setting',  # Should almost always be "{group}__{dest}"
             'metavar': 'TEST_SETTING',  # Set manually so argparse doesn't use TST_TEST
             'nargs': None,
             'required': None,
@@ -94,7 +96,7 @@ success = [
                 'choices': None,
                 'const': None,
                 'default': None,
-                'dest': 'tst_test_setting',
+                'dest': 'tst__test_setting',
                 'help': None,
                 'metavar': 'TEST_SETTING',
                 'nargs': None,
@@ -117,13 +119,15 @@ success = [
             'cmdline': True,
             'const': None,
             'default': None,
+            'setting_name': 'test',  # setting_name is calculated by Setting and is not used by argparse
             'dest': 'testing',  # dest is calculated by Setting and is not used by argparse
             'display_name': 'testing',  # defaults to dest
             'exclusive': False,
             'file': True,
+            'flag': True,
             'group': 'tst',
             'help': None,
-            'internal_name': 'tst_testing',  # Should almost always be "{group}_{dest}"
+            'internal_name': 'tst__testing',  # Should almost always be "{group}__{dest}"
             'metavar': 'TESTING',  # Set manually so argparse doesn't use TST_TEST
             'nargs': None,
             'required': None,
@@ -134,7 +138,7 @@ success = [
                 'choices': None,
                 'const': None,
                 'default': None,
-                'dest': 'tst_testing',
+                'dest': 'tst__testing',
                 'help': None,
                 'metavar': 'TESTING',
                 'nargs': None,
@@ -156,13 +160,15 @@ success = [
             'cmdline': True,
             'const': None,
             'default': None,
+            'setting_name': 'test',  # dest is calculated by Setting and is not used by argparse
             'dest': 'test',  # dest is calculated by Setting and is not used by argparse
             'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
+            'flag': True,
             'group': 'tst',
             'help': None,
-            'internal_name': 'tst_test',  # Should almost always be "{group}_{dest}"
+            'internal_name': 'tst__test',  # Should almost always be "{group}__{dest}"
             'metavar': 'TEST',  # Set manually so argparse doesn't use TST_TEST
             'nargs': None,
             'required': None,
@@ -173,7 +179,7 @@ success = [
                 'choices': None,
                 'const': None,
                 'default': None,
-                'dest': 'tst_test',
+                'dest': 'tst__test',
                 'help': None,
                 'metavar': 'TEST',
                 'nargs': None,
@@ -196,13 +202,15 @@ success = [
             'cmdline': True,
             'const': None,
             'default': None,
+            'setting_name': 'test',  # dest is calculated by Setting and is not used by argparse
             'dest': 'test',  # dest is calculated by Setting and is not used by argparse
             'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
+            'flag': True,
             'group': 'tst',
             'help': None,
-            'internal_name': 'tst_test',  # Should almost always be "{group}_{dest}"
+            'internal_name': 'tst__test',  # Should almost always be "{group}__{dest}"
             'metavar': None,  # store_true does not get a metavar
             'nargs': None,
             'required': None,
@@ -213,7 +221,7 @@ success = [
                 'choices': None,
                 'const': None,
                 'default': None,
-                'dest': 'tst_test',
+                'dest': 'tst__test',
                 'help': None,
                 'metavar': None,
                 'nargs': None,
@@ -235,13 +243,15 @@ success = [
             'cmdline': True,
             'const': None,
             'default': None,
+            'setting_name': 'test',
             'dest': 'test',
             'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
+            'flag': True,
             'group': 'tst',
             'help': None,
-            'internal_name': 'tst_test',
+            'internal_name': 'tst__test',
             'metavar': 'TEST',
             'nargs': None,
             'required': None,
@@ -252,7 +262,7 @@ success = [
                 'choices': None,
                 'const': None,
                 'default': None,
-                'dest': 'tst_test',
+                'dest': 'tst__test',
                 'help': None,
                 'metavar': 'TEST',
                 'nargs': None,
@@ -274,18 +284,20 @@ success = [
             'cmdline': True,
             'const': None,
             'default': None,
+            'setting_name': 'test',
             'dest': 'test',
             'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
+            'flag': False,
             'group': 'tst',
             'help': None,
-            'internal_name': 'tst_test',
+            'internal_name': 'tst__test',
             'metavar': 'TEST',
             'nargs': None,
             'required': None,
             'type': None,
-            'argparse_args': ('tst_test',),
+            'argparse_args': ('tst__test',),
             'argparse_kwargs': {
                 'action': None,
                 'choices': None,
@@ -311,10 +323,12 @@ success = [
             'cmdline': True,
             'const': None,
             'default': None,
+            'setting_name': 'test',
             'dest': 'test',
             'display_name': 'test',  # defaults to dest
             'exclusive': False,
             'file': True,
+            'flag': True,
             'group': '',
             'help': None,
             'internal_name': 'test',  # No group, leading _ is stripped

tests/settngs_test.py

@@ -6,6 +6,7 @@ import json
 import pathlib
 import sys
 from collections import defaultdict
+from textwrap import dedent
 from typing import Generator
 
 import pytest
@@ -17,10 +18,40 @@ from testing.settngs import failure
 from testing.settngs import success
 
 
-if sys.version_info < (3, 9):  # pragma: no cover
+if sys.version_info >= (3, 10):  # pragma: no cover
+    List = list
+    help_output = '''\
+usage: __main__.py [-h] [TEST ...]
+
+positional arguments:
+  TEST
+
+options:
+  -h, --help  show this help message and exit
+'''
+elif sys.version_info < (3, 9):  # pragma: no cover
     from typing import List
+    help_output = '''\
+usage: __main__.py [-h] [TEST [TEST ...]]
+
+positional arguments:
+  TEST
+
+optional arguments:
+  -h, --help  show this help message and exit
+'''
+
 else:  # pragma: no cover
     List = list
+    help_output = '''\
+usage: __main__.py [-h] [TEST ...]
+
+positional arguments:
+  TEST
+
+optional arguments:
+  -h, --help  show this help message and exit
+'''
 
 
 @pytest.fixture
@@ -63,6 +94,55 @@ def test_add_setting(settngs_manager):
     assert settngs_manager.add_setting('--test') is None
 
 
+def test_add_setting_invalid_name(settngs_manager):
+    with pytest.raises(Exception, match='Cannot use test¥ in a namespace'):
+        assert settngs_manager.add_setting('--test¥') is None
+
+
+def test_sub_group(settngs_manager):
+    with pytest.raises(Exception, match='Sub groups are not allowed'):
+        settngs_manager.add_group('tst', lambda parser: parser.add_group('tst', lambda parser: parser.add_setting('--test2', default='hello')))
+
+
+def test_sub_persistent_group(settngs_manager):
+    with pytest.raises(Exception, match='Sub groups are not allowed'):
+        settngs_manager.add_persistent_group('tst', lambda parser: parser.add_persistent_group('tst', lambda parser: parser.add_setting('--test2', default='hello')))
+
+
+def test_redefine_persistent_group(settngs_manager):
+    settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test2', default='hello'))
+    with pytest.raises(Exception, match='Group already exists and is not persistent'):
+        settngs_manager.add_persistent_group('tst', None)
+
+
+def test_exclusive_group(settngs_manager):
+    settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'), exclusive_group=True)
+    settngs_manager.create_argparser()
+    args = settngs_manager.argparser.parse_args(['--test', 'never'])
+    assert args.tst__test == 'never'
+
+    with pytest.raises(SystemExit):
+        settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test2', default='hello'), exclusive_group=True)
+        settngs_manager.create_argparser()
+        args = settngs_manager.argparser.parse_args(['--test', 'never', '--test2', 'never'])
+
+
+def test_files_group(capsys, settngs_manager):
+    settngs_manager.add_group('runtime', lambda parser: parser.add_setting('test', default='hello', nargs='*'))
+    settngs_manager.create_argparser()
+    settngs_manager.argparser.print_help()
+    captured = capsys.readouterr()
+    assert captured.out == help_output
+
+
+def test_setting_without_group(capsys, settngs_manager):
+    settngs_manager.add_setting('test', default='hello', nargs='*')
+    settngs_manager.create_argparser()
+    settngs_manager.argparser.print_help()
+    captured = capsys.readouterr()
+    assert captured.out == help_output
+
+
 class TestValues:
 
     def test_invalid_normalize(self, settngs_manager):
@@ -130,6 +210,24 @@ class TestValues:
         assert non_defaults_normalized.values['tst'] == {'test': 'world'}
         assert non_defaults_normalized.values['tst_persistent'] == {'test': 'world'}
 
+    def test_normalize_dest(self, settngs_manager):
+        settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
+        settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test2', dest='test', default='hello'))
+        settngs_manager.add_persistent_group('tst_persistent', lambda parser: parser.add_setting('--test', default='hello'))
+
+        defaults = settngs_manager.defaults()
+        defaults_normalized = settngs_manager.normalize_config(defaults, file=True, default=False)
+        assert defaults_normalized.values['tst'] == {}
+        assert defaults_normalized.values['tst_persistent'] == {}
+
+        non_defaults = settngs_manager.defaults()
+        non_defaults.values['tst']['test'] = 'world'
+        non_defaults.values['tst_persistent']['test'] = 'world'
+        non_defaults_normalized = settngs_manager.normalize_config(non_defaults, file=True, default=False)
+
+        assert non_defaults_normalized.values['tst'] == {'test': 'world'}
+        assert non_defaults_normalized.values['tst_persistent'] == {'test': 'world'}
+
     def test_normalize(self, settngs_manager):
         settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
         settngs_manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
@@ -147,6 +245,29 @@ class TestValues:
         assert normalized['persistent']['hello'] == 'success'
         assert normalized['persistent']['world'] == 'world'
 
+    def test_unknown_group(self):
+        manager = settngs.Manager()
+        manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
+        manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
+
+        manager_unknown = settngs.Manager()
+        manager_unknown.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
+        # This manager doesn't know about this group
+        # manager_unknown.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
+
+        defaults = manager.defaults()
+        defaults.values['test'] = 'fail'  # type: ignore[assignment] # Not defined in manager, should be removed
+        defaults.values['persistent']['hello'] = 'success'  # Group is not defined in manager_unknown, should stay
+
+        normalized, _ = manager_unknown.normalize_config(defaults.values, file=True)
+
+        assert 'test' not in normalized
+        assert 'tst' in normalized
+        assert 'test' in normalized['tst']
+        assert normalized['tst']['test'] == 'hello'
+        assert normalized['persistent']['hello'] == 'success'
+        assert normalized['persistent']['world'] == 'world'
+
 
 class TestNamespace:
 
@@ -164,12 +285,12 @@ class TestNamespace:
     def test_get_defaults_group(self, settngs_manager):
         settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
         defaults, _ = settngs_manager.get_namespace(settngs_manager.defaults(), file=True, cmdline=True)
-        assert defaults.tst_test == 'hello'
+        assert defaults.tst__test == 'hello'
 
     def test_get_defaults_group_space(self, settngs_manager):
         settngs_manager.add_group('Testing tst', lambda parser: parser.add_setting('--test', default='hello'))
         defaults, _ = settngs_manager.get_namespace(settngs_manager.defaults(), file=True, cmdline=True)
-        assert defaults.Testing_tst_test == 'hello'
+        assert defaults.Testing_tst__test == 'hello'
 
     def test_cmdline_only(self, settngs_manager):
         settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello', file=False))
@@ -178,11 +299,11 @@ class TestNamespace:
         file_normalized, _ = settngs_manager.get_namespace(settngs_manager.normalize_config(settngs_manager.defaults(), file=True), file=True)
         cmdline_normalized, _ = settngs_manager.get_namespace(settngs_manager.normalize_config(settngs_manager.defaults(), cmdline=True), cmdline=True)
 
-        assert 'tst_test' not in file_normalized.__dict__
-        assert 'tst2_test2' in file_normalized.__dict__
+        assert 'tst__test' not in file_normalized.__dict__
+        assert 'tst2__test2' in file_normalized.__dict__
 
-        assert 'tst_test' in cmdline_normalized.__dict__
-        assert 'tst2_test2' not in cmdline_normalized.__dict__
+        assert 'tst__test' in cmdline_normalized.__dict__
+        assert 'tst2__test2' not in cmdline_normalized.__dict__
 
     def test_cmdline_only_persistent_group(self, settngs_manager):
         settngs_manager.add_persistent_group('tst', lambda parser: parser.add_setting('--test', default='hello', file=False))
@@ -191,11 +312,11 @@ class TestNamespace:
         file_normalized, _ = settngs_manager.get_namespace(settngs_manager.normalize_config(settngs_manager.defaults(), file=True), file=True)
         cmdline_normalized, _ = settngs_manager.get_namespace(settngs_manager.normalize_config(settngs_manager.defaults(), cmdline=True), cmdline=True)
 
-        assert 'tst_test' not in file_normalized.__dict__
-        assert 'tst2_test2' in file_normalized.__dict__
+        assert 'tst__test' not in file_normalized.__dict__
+        assert 'tst2__test2' in file_normalized.__dict__
 
-        assert 'tst_test' in cmdline_normalized.__dict__
-        assert 'tst2_test2' not in cmdline_normalized.__dict__
+        assert 'tst__test' in cmdline_normalized.__dict__
+        assert 'tst2__test2' not in cmdline_normalized.__dict__
 
     def test_normalize_defaults(self, settngs_manager):
         settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
@@ -207,12 +328,29 @@ class TestNamespace:
         assert defaults_normalized.values.__dict__ == {}
 
         non_defaults = settngs_manager.get_namespace(settngs_manager.defaults(), file=True, cmdline=True)
-        non_defaults.values.tst_test = 'world'
-        non_defaults.values.tst_persistent_test = 'world'
+        non_defaults.values.tst__test = 'world'
+        non_defaults.values.tst_persistent__test = 'world'
         non_defaults_normalized = settngs_manager.get_namespace(settngs_manager.normalize_config(non_defaults, file=True, default=False), file=True, default=False)
 
-        assert non_defaults_normalized.values.tst_test == 'world'
-        assert non_defaults_normalized.values.tst_persistent_test == 'world'
+        assert non_defaults_normalized.values.tst__test == 'world'
+        assert non_defaults_normalized.values.tst_persistent__test == 'world'
+
+    def test_normalize_dest(self, settngs_manager):
+        settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
+        settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test2', dest='test', default='hello'))
+        settngs_manager.add_persistent_group('tst_persistent', lambda parser: parser.add_setting('--test', default='hello'))
+
+        defaults = settngs_manager.defaults()
+        defaults_normalized = settngs_manager.get_namespace(settngs_manager.normalize_config(defaults, file=True, default=False), file=True, default=False)
+        assert defaults_normalized.values.__dict__ == {}
+
+        non_defaults = settngs_manager.get_namespace(settngs_manager.defaults(), file=True, cmdline=True)
+        non_defaults.values.tst__test = 'world'
+        non_defaults.values.tst_persistent__test = 'world'
+        non_defaults_normalized = settngs_manager.get_namespace(settngs_manager.normalize_config(non_defaults, file=True, default=False), file=True, default=False)
+
+        assert non_defaults_normalized.values.tst__test == 'world'
+        assert non_defaults_normalized.values.tst_persistent__test == 'world'
 
     def test_normalize(self, settngs_manager):
         settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
@@ -220,15 +358,37 @@ class TestNamespace:
 
         defaults = settngs_manager.get_namespace(settngs_manager.defaults(), file=True, cmdline=True)
         defaults.values.test = 'fail'  # Not defined in settngs_manager, should be removed
-        defaults.values.persistent_hello = 'success'  # Not defined in settngs_manager, should stay
+        defaults.values.persistent__hello = 'success'  # Not defined in settngs_manager, should stay
 
         normalized, _ = settngs_manager.get_namespace(settngs_manager.normalize_config(defaults, file=True), file=True)
 
         assert not hasattr(normalized, 'test')
-        assert hasattr(normalized, 'tst_test')
-        assert normalized.tst_test == 'hello'
-        assert normalized.persistent_hello == 'success'
-        assert normalized.persistent_world == 'world'
+        assert hasattr(normalized, 'tst__test')
+        assert normalized.tst__test == 'hello'
+        assert normalized.persistent__hello == 'success'
+        assert normalized.persistent__world == 'world'
+
+    def test_normalize_unknown_group(self, settngs_manager):
+        manager = settngs.Manager()
+        manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
+        manager.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
+
+        manager_unknown = settngs.Manager()
+        manager_unknown.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
+        # This manager doesn't know about this group
+        # manager_unknown.add_persistent_group('persistent', lambda parser: parser.add_setting('--world', default='world'))
+
+        defaults = manager.get_namespace(manager.defaults(), file=True, cmdline=True)
+        defaults.values.test = 'fail'  # Not defined in manager, should be removed
+        defaults.values.persistent__hello = 'success'  # Not defined in manager, should stay
+
+        normalized, _ = manager_unknown.get_namespace(defaults.values, file=True)
+
+        assert not hasattr(normalized, 'test')
+        assert hasattr(normalized, 'tst__test')
+        assert normalized.tst__test == 'hello'
+        assert normalized.persistent__hello == 'success'
+        assert normalized.persistent__world == 'world'
 
 
 def test_get_namespace_with_namespace(settngs_manager):
@@ -240,7 +400,7 @@ def test_get_namespace_with_namespace(settngs_manager):
 def test_get_namespace_group(settngs_manager):
     settngs_manager.add_group('tst', lambda parser: parser.add_setting('--test', default='hello'))
     defaults, _ = settngs_manager.get_namespace(settngs_manager.defaults(), file=True)
-    assert defaults.tst_test == 'hello'
+    assert defaults.tst__test == 'hello'
 
 
 def test_clean_config(settngs_manager):
@@ -271,8 +431,8 @@ def test_parse_cmdline(settngs_manager):
 
 namespaces = (
     lambda definitions: settngs.Config({'tst': {'test': 'fail', 'test2': 'success'}}, definitions),
-    lambda definitions: settngs.Config(argparse.Namespace(tst_test='fail', tst_test2='success'), definitions),
-    lambda definitions: argparse.Namespace(tst_test='fail', tst_test2='success'),
+    lambda definitions: settngs.Config(argparse.Namespace(tst__test='fail', tst__test2='success'), definitions),
+    lambda definitions: argparse.Namespace(tst__test='fail', tst__test2='success'),
 )
 
 
@@ -533,6 +693,7 @@ settings = (
     (lambda parser: parser.add_setting('-t', '--test', action='help'), None),
     (lambda parser: parser.add_setting('-t', '--test', action='version'), None),
     (lambda parser: parser.add_setting('-t', '--test', type=int), 'int'),
+    (lambda parser: parser.add_setting('-t', '--test', nargs='+'), List[str]),
     (lambda parser: parser.add_setting('-t', '--test', type=_typed_function), 'tests.settngs_test.test_type'),
     (lambda parser: parser.add_setting('-t', '--test', type=_untyped_function, default=1), 'int'),
     (lambda parser: parser.add_setting('-t', '--test', type=_untyped_function), 'typing.Any'),
@@ -543,16 +704,21 @@ settings = (
 def test_generate_ns(settngs_manager, set_options, typ):
     settngs_manager.add_group('test', set_options)
 
-    src = '''\
+    src = dedent('''\
     from __future__ import annotations
-import typing
+
     import settngs
-'''
+    ''')
+
+    if 'typing.' in str(typ):
+        src += '\nimport typing'
     if typ == 'tests.settngs_test.test_type':
-        src += 'import tests.settngs_test\n'
-    src += '''
+        src += '\nimport tests.settngs_test'
+    src += dedent('''
+
+
     class settngs_namespace(settngs.TypedNS):
-'''
+    ''')
     if typ is None:
         src += '    ...\n'
     else: