feat: sketch of a new configuration manager

This will eventually fix
datalad#397

datalad_next/config/__init__.py

@@ -9,4 +9,36 @@
    ConfigManager
 """
 
+__all__ = [
+    'ConfigManager',
+    'ConfigurationSource',
+    'Environment',
+    'ImplementationDefault',
+    'MultiConfiguration',
+    'defaults',
+    'dialog',
+]
+
+# TODO: eventually replace with
+# from .legacy import ConfigManager
 from datalad.config import ConfigManager
+
+from . import dialog
+from .default import (
+    ImplementationDefault,
+)
+from .env import Environment
+from .legacy import ConfigManager as LegacyConfigManager
+from .multi import MultiConfiguration
+from .source import ConfigurationSource
+
+# instance for registering all defaults
+defaults = ImplementationDefault()
+
+manager = MultiConfiguration({
+    # order reflects precedence rule, first source with a
+    # key takes precedence
+    'environment': Environment(),
+    'defaults': defaults,
+})
+)

datalad_next/config/default.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import logging
+from typing import (
+    Any,
+    Callable,
+)
+
+
+from datalad_next.config.dialog import get_dialog_class_from_legacy_ui_label
+from datalad_next.config.item import ConfigurationItem
+from datalad_next.config.source import ConfigurationSource
+from datalad_next.constraints import (
+    Constraint,
+    DatasetParameter,
+    NoConstraint,
+)
+
+lgr = logging.getLogger('datalad.config.default')
+
+
+class ImplementationDefault(ConfigurationSource):
+    is_writable = True
+
+    def load(self) -> None:
+        # there is no loading. clients have to set any items they want to
+        # see a default known for. There is typically only one instance of
+        # this class, and it is the true source of the information by itself.
+        pass
+
+    def __setitem__(self, key: str, value: ConfigurationItem) -> None:
+        if key in self:
+            # resetting is something that is an unusual event.
+            # __setitem__ does not allow for a dedicated "force" flag,
+            # so we leave a message at least
+            lgr.debug('Resetting %r default', key)
+        super().__setitem__(key, value)
+
+    def __str__(self):
+        return 'ImplementationDefaults'

datalad_next/config/dialog.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+__all__ = [
+    'Dialog',
+    'Question',
+    'YesNo',
+    'Choice',
+]
+
+
+@dataclass(kw_only=True)
+class Dialog:
+    title: str
+    text: str
+
+
+@dataclass(kw_only=True)
+class Question(Dialog):
+    pass
+
+
+@dataclass(kw_only=True)
+class YesNo(Dialog):
+    pass
+
+
+@dataclass(kw_only=True)
+class Choice(Dialog):
+    pass

datalad_next/config/env.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import json
+import logging
+from os import environ
+from typing import Any
+
+from datalad_next.config.item import ConfigurationItem
+from datalad_next.config.source import ConfigurationSource
+
+lgr = logging.getLogger('datalad.config')
+
+
+class Environment(ConfigurationSource):
+    """
+    All loaded items have a `store_target` of `Environment, assuming
+    that if they are loaded from the environment, a modification can
+    also target the environment again.
+    """
+    is_writable = True
+
+    def load(self) -> None:
+        # clean slate
+        self.reset()
+        for k in environ:
+            if not k.startswith('DATALAD_'):
+                continue
+            # translate variable name to config item key
+            item_key = k.replace('__', '-').replace('_', '.').lower()
+            self[item_key] = ConfigurationItem(
+                value=environ[k],
+                store_target=Environment,
+            )
+
+    def __str__(self):
+        return 'Environment'

datalad_next/config/item.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+)
+
+if TYPE_CHECKING:
+    from datalad_next.config import (
+        ConfigurationSource,
+    )
+    from datalad_next.config import (
+        dialog as dialog_collection,
+    )
+    from datalad_next.constraints import Constraint
+
+
+@dataclass
+class ConfigurationItem:
+    value: Any
+    """Value of a configuration item"""
+    validator: Constraint | Callable | None = None
+    """Type or validator of the configuration value"""
+    dialog: dialog_collection.Dialog | None = None
+    """Hint how a UI should gather a value for this item"""
+    store_target: type[ConfigurationSource] | str | None = None
+    """Hint with which configuration source this item should be stored
+
+    Any hint should be a type.
+
+    If a string label is given, it will be interpreted as a class name.  This
+    functionality is deprecated and is only supported, for the time being, to
+    support legacy implementations. It should not be used for any new
+    implementations.
+    """
+
+    # TODO: `default_fn` would be a validator that returns
+    # the final value from a possibly pointless value like
+    # None -- in an ImplementationDefault(ConfigurationSource)

datalad_next/config/multi.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from itertools import chain
+from types import MappingProxyType
+from typing import (
+    Any,
+    TYPE_CHECKING,
+)
+
+if TYPE_CHECKING:
+    from datalad_next.config import (
+        ConfigurationItem,
+        ConfigurationSource,
+    )
+
+
+class MultiConfiguration:
+    """Query different sources of configuration settings
+
+    This is query-centered. Manipulation is supported by
+    by individual configuration source implementations.
+    This separation is done for two reasons. 1) Query is
+    a much more frequent operation than write, and
+    2) consolidating different sources for read is sensible,
+    and doable, while a uniform semantics and behavior for
+    write are complicated due to the inherent differences
+    across sources.
+    """
+    def __init__(
+        self,
+        sources: dict[str, ConfigurationSource],
+    ):
+        # we keep the sources strictly separate.
+        # the order here matters and represents the
+        # precedence rule
+        self._sources = sources
+
+    @property
+    def sources(self) -> MappingProxyType:
+        return MappingProxyType(self._sources)
+
+    def __len__(self):
+        return len(self.keys())
+
+    def __getitem__(self, key) -> ConfigurationItem:
+        for s in self._sources.values():
+            if key in s:
+                return s[key]
+        raise KeyError
+
+    def getvalue(self, key, default: Any = None) -> Any:
+        # TODO: consider on-access validation using a validator that
+        # is possibly registered in another source
+        # TODO: consolidate validation behavior with
+        # ConfigurationSource.getvalue()
+        for s in self._sources.values():
+            if key in s:
+                return s.getvalue(key)
+        return default
+
+    def __contains__(self, key):
+        for _, s in self._sources.items():
+            if key in s:
+                return True
+        return False
+
+    def keys(self) -> set[str]:
+        return set(chain.from_iterable(s.keys()
+                                       for s in self._sources.values()))

datalad_next/config/source.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from abc import (
+    ABC,
+    abstractmethod,
+)
+from typing import (
+    Any,
+    KeysView,
+)
+
+from datalad_next.config.item import ConfigurationItem
+
+
+class ConfigurationSource(ABC):
+    def __init__(
+        self,
+        load=True,  # noqa: FBT002
+    ):
+        self.reset()
+        if load:
+            self.load()
+
+    @property
+    @abstractmethod
+    def is_writable(self) -> bool:
+        """Flag whether configuration item values can be set at the source"""
+
+    @abstractmethod
+    def load(self) -> None:
+        """Implements loading items from the configuration source.
+
+        It is expected that after calling this method, an instance of
+        this source reports on configuration items according to the
+        latest/current state of the source.
+
+        No side-effects are implied. Particular implementations may
+        even choose to have this method be a no-op.
+        """
+
+    def reset(self) -> None:
+        # particular implementations may not use this facility,
+        # but it is provided as a convenience. Maybe factor
+        # it out into a dedicated subclass even.
+        self._items: dict[str, ConfigurationItem] = {}
+
+    def __len__(self) -> int:
+        return len(self._items)
+
+    def __getitem__(self, key: str) -> ConfigurationItem:
+        return self._items[key]
+
+    def __setitem__(self, key: str, value: ConfigurationItem) -> None:
+        if not self.is_writable:
+            raise NotImplementedError
+        self._items[key] = value
+
+    def __contains__(self, key: str) -> bool:
+        return key in self._items
+
+    def keys(self) -> KeysView:
+        return self._items.keys()
+
+    def get(self, key, default: Any = None) -> ConfigurationItem:
+        try:
+            return self._items[key]
+        except KeyError:
+            if isinstance(default, ConfigurationItem):
+                return default
+            return ConfigurationItem(value=default)
+
+    def getvalue(self, key, default: Any = None) -> Any:
+        if key not in self:
+            return default
+        item = self[key]
+        # there are two ways to do validation and type conversion.
+        # on-access, or on-load. Doing it on-load would allow to reject
+        # invalid configuration immediately. But it might spend time
+        # on items that never get accessed. On-access might waste
+        # cycles on repeated checks, and possible complain later than
+        # useful. Here we nevertheless run a validator on-access in
+        # the default implementation. Particular sources may want to
+        # override this, or ensure that the stored value that is passed
+        # to a validator is already in the best possible form to
+        # make re-validation the cheapest.
+        return item.validator(item.value) if item.validator else item.value
+
+    def __repr__(self) -> str:
+        return self._items.__repr__()
+
+    def __str__(self) -> str:
+        return self._items.__str__()

datalad_next/config/tests/test_env.py

@@ -0,0 +1,12 @@
+from ..env import Environment
+
+
+def test_load_datalad_env(monkeypatch):
+    target_key = 'datalad.chunky-monkey.feedback'
+    target_value = 'ohmnomnom'
+    with monkeypatch.context() as m:
+        m.setenv('DATALAD_CHUNKY__MONKEY_FEEDBACK', 'ohmnomnom')
+        env = Environment()
+        assert target_key in env.keys()  # noqa: SIM118
+        assert target_key in env
+        assert env.getvalue(target_key) == target_value