[WIP] Dynaconf for yamls

cfme/fixtures/parallelizer/__init__.py

@@ -57,11 +57,11 @@
 
 # Initialize slaveid to None, indicating this as the master process
 # slaves will set this to a unique string when they're initialized
-conf.runtime['env']['slaveid'] = None
+conf['env']['slaveid'] = None
 
-if not conf.runtime['env'].get('ts'):
+if not conf['env'].get('ts'):
     ts = str(time())
-    conf.runtime['env']['ts'] = ts
+    conf['env']['ts'] = ts
 
 
 def pytest_addhooks(pluginmanager):
@@ -127,7 +127,7 @@ def start(self):
             sys.executable, remote.__file__,
             '--worker', self.id,
             '--appliance', self.appliance.as_json,
-            '--ts', conf.runtime['env']['ts'],
+            '--ts', conf['env']['ts'],
             '--config', json.dumps(self.worker_config),
 
 

cfme/fixtures/parallelizer/remote.py

@@ -19,9 +19,8 @@ def __init__(self, config, slaveid, zmq_endpoint):
         self.config = config
         self.session = None
         self.collection = None
-        self.slaveid = conf.runtime['env']['slaveid'] = slaveid
+        self.slaveid = conf['env']['slaveid'] = slaveid
         self.log = cfme.utils.log.logger
-        conf.clear()
         # Override the logger in utils.log
 
         ctx = zmq.Context.instance()
@@ -241,8 +240,8 @@ def _init_config(slave_options, slave_args):
     from cfme.fixtures.pytest_store import store
     from cfme.utils import conf
 
-    conf.runtime['env']['slaveid'] = args.worker
-    conf.runtime['env']['ts'] = args.ts
+    conf['env']['slaveid'] = args.worker
+    conf['env']['ts'] = args.ts
     store.parallelizer_role = 'slave'
 
     slave_args = config.pop('args')
@@ -251,8 +250,8 @@ def _init_config(slave_options, slave_args):
     appliance_data = config["appliance_data"]
     if ip_address in appliance_data:
         template_name, provider_name = appliance_data[ip_address]
-        conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
-        conf.runtime["cfme_data"]["basic_info"]["appliances_provider"] = provider_name
+        conf["cfme_data"]["basic_info"]["appliance_template"] = template_name
+        conf["cfme_data"]["basic_info"]["appliances_provider"] = provider_name
     pytest_config = _init_config(slave_options, slave_args)
     slave_manager = SlaveManager(pytest_config, args.worker, config['zmq_endpoint'])
     pytest_config.pluginmanager.register(slave_manager, 'slave_manager')

cfme/markers/env_markers/provider.py

@@ -189,6 +189,7 @@ def all_required(miq_version, filters=None):
     try:
         data_for_stream = conf.supportability[stream]['providers']
     except KeyError:
+        logger.warning(f"A KeyError was caught while accessing supportability for {stream}")
         # there are cases when such data isn't available. For instance travis
         data_for_stream = {}
 

cfme/test_framework/sprout/plugin.py

@@ -188,7 +188,7 @@ def mangle_in_sprout_appliances(config):
         log.info("- %s is %s", appliance['url'], appliance['name'])
     mgr.reset_timer()
     template_name = requested_appliances[0]["template_name"]
-    conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
+    conf["cfme_data"]["basic_info"]["appliance_template"] = template_name
     log.info("appliance_template: %s", template_name)
     with project_path.join('.appliance_template').open('w') as template_file:
         template_file.write('export appliance_template="{}"'.format(template_name))

cfme/utils/conf.py

@@ -1,12 +1,12 @@
 import sys
 
-from cfme.test_framework.config import DeprecatedConfigWrapper
-from cfme.test_framework.config import global_configuration
 from cfme.utils import path
+from cfme.utils.config import ConfigWrapper
+from cfme.utils.config import global_configuration
 
-global_configuration.configure(
+global_configuration.configure_creds(
     config_dir=path.conf_path.strpath,
-    crypt_key_file=path.project_path.join('.yaml_key').strpath,
+    crypt_key_file=path.project_path.join('.yaml_key').strpath
 )
 
-sys.modules[__name__] = DeprecatedConfigWrapper(global_configuration)
+sys.modules[__name__] = ConfigWrapper(global_configuration)

cfme/utils/config.py

@@ -0,0 +1,140 @@
+"""
+classes to manage the cfme test framework configuration
+"""
+import os
+
+import attr
+import yaycl
+from cached_property import cached_property
+from dynaconf import LazySettings
+
+from cfme.utils import path
+
+YAML_KEYS = ['auth_data', 'cfme_data', 'cfme_performance', 'composite',
+            'credentials', 'docker', 'env', 'gpg', 'hidden', 'jenkins',
+            'migration_tests', 'perf_tests', 'polarion_tools', 'rdb',
+            'supportability']
+
+
+class ConfigData(object):
+    """A wrapper for configs in yamls- reads and loads values from yaml files in conf/ dir."""
+    def __init__(self, env='cfme_data'):
+        self.settings = LazySettings(
+            ENV_FOR_DYNACONF=env,
+            INCLUDES_FOR_DYNACONF='{}/*.yaml'.format(path.conf_path.strpath),
+            ROOT_PATH_FOR_DYNACONF=path.conf_path.strpath
+        )
+
+    def __getitem__(self, key):
+        return self.__getattr__(key)
+
+    def __getattr__(self, key):
+        self.config = self.settings[key.upper()]
+        return self.config
+
+    def __setitem__(self, key, value):
+        """This function allows you to mimic some behavior like runtime from yaycl
+            You can do following:
+                from cfme.utils import conf
+                conf ['env']['new_key'] = 'new_val'
+                conf ['env']['new_key']
+                # The new_key and its value is cached through the session
+        """
+        self.settings[key] = value
+
+    def __contains__(self, key):
+        return key in self.settings.as_dict() or key.upper() in self.settings.as_dict()
+
+    def get(self, key, default=None):
+        try:
+            return self.__getattr__(key)
+        except KeyError:
+            return default
+
+
+class Configuration(object):
+    """
+    Holds ConfigData objects in dictionary `settings`.
+    """
+    def __init__(self):
+        self.yaycl_config = None
+
+    def configure_creds(self, config_dir, crypt_key_file=None):
+        """
+        do the defered initial loading of the configuration
+
+        :param config_dir: path to the folder with configuration files
+        :param crypt_key_file: optional name of a file holding the key for encrypted
+            configuration files
+
+        :raises: AssertionError if called more than once
+
+        if the `utils.conf` api is removed, the loading can be transformed to eager loading
+        """
+        assert self.yaycl_config is None
+        if crypt_key_file and os.path.exists(crypt_key_file):
+            self.yaycl_config = yaycl.Config(
+                config_dir=config_dir,
+                crypt_key_file=crypt_key_file)
+        else:
+            self.yaycl_config = yaycl.Config(config_dir=config_dir)
+
+    @cached_property
+    def settings(self):
+        """
+        Loads all the congfigs in settings dictionary where Key is filename(without extension)
+        and value is ConfigData object relevant to that file.
+
+        Example self.settings would look like:
+        {'cfme_data': <cfme.utils.config_copy.ConfigData at 0x7f2a943b14e0>,
+         'env': <cfme.utils.config_copy.ConfigData at 0x7f2a943b1be0>,
+         'gpg': <cfme.utils.config_copy.ConfigData at 0x7f2a943b1cc0>,
+         ...
+         ...
+         'polarion_tools': <cfme.utils.config_copy.ConfigData at 0x7f2a943ad438>,
+         'perf_tests': <cfme.utils.config_copy.ConfigData at 0x7f2a943ad2e8>}
+
+        """
+        settings = {
+            f.purebasename: ConfigData(env=f.purebasename)
+            for f in path.conf_path.listdir() if '.local.yaml' not in f.basename and
+            f.ext in ['.yaml', '.yml']
+        }
+        return settings
+
+    def get_config(self, name):
+        """returns a config object
+
+        :param name: name of the configuration object
+        """
+        if name in YAML_KEYS:
+            if name == 'credentials' or name == 'hidden':
+                return getattr(self.yaycl_config, name)
+            try:
+                return self.settings[name]
+            except KeyError:
+                # seems like config was deleted, reload
+                self.settings[name] = ConfigData(env=name)
+                return self.settings[name]
+
+    def del_config(self, name):
+        if self.settings:
+            if name in self.settings:
+                del self.settings[name]
+
+
+@attr.s
+class ConfigWrapper(object):
+    configuration = attr.ib()
+
+    def __getattr__(self, key):
+        return self.configuration.get_config(key)
+
+    def __getitem__(self, key):
+        return self.configuration.get_config(key)
+
+    def __delitem__(self, key):
+        return self.configuration.del_config(key)
+
+
+global_configuration = Configuration()

cfme/utils/log.py

@@ -220,13 +220,6 @@ def process(self, message, kwargs):
 
 
 def _load_conf(logger_name=None):
-    # Reload logging conf from env, then update the logging_conf
-    try:
-        del(conf['env'])
-    except KeyError:
-        # env not loaded yet
-        pass
-
     logging_conf = _default_conf.copy()
 
     yaml_conf = conf.env.get('logging', {})