WIP yaml support

checkbox-ng/plainbox/impl/secure/providers/v1.py

@@ -27,6 +27,7 @@
 import gettext
 import logging
 import os
+import yaml
 
 from plainbox.abc import IProvider1
 from plainbox.i18n import gettext as _
@@ -48,6 +49,7 @@
 from plainbox.impl.secure.rfc822 import FileTextSource
 from plainbox.impl.secure.rfc822 import RFC822SyntaxError
 from plainbox.impl.secure.rfc822 import load_rfc822_records
+from plainbox.impl.secure.yaml import load_yaml_records
 from plainbox.impl.unit import all_units
 from plainbox.impl.unit.file import FileRole
 from plainbox.impl.unit.file import FileUnit
@@ -278,6 +280,96 @@ def _get_unit_cls(unit_name):
         all_units.load()
         return all_units.get_by_name(unit_name).plugin_object
 
+class YamlUnitPlugIn(ProviderContentPlugIn):
+    """
+    Loader of Checkbox units encoded in YAML.
+    """
+    def inspect(
+        self,
+        filename: str,
+        text: str,
+        provider: "Provider1",
+        validate: bool,
+        validation_kwargs: "Dict[str, Any]",
+        check: bool,
+        context: "???",
+    ) -> "Any":
+        """
+        Load all units from the YAML file.
+        """
+        logger.debug(_("Loading units from %r..."), filename)
+        try:
+            records = load_yaml_records(text, source=FileTextSource(filename))
+        except yaml.YAMLError as exc:
+            raise PlugInError(
+                _("Cannot load job definitions from {!r}: {}").format(
+                    filename, exc
+                )
+            )
+        unit_list = []
+        for record in records:
+            unit_name = record.data.get("unit", "job")
+            try:
+                unit_cls = self._get_unit_cls(unit_name)
+            except KeyError:
+                raise PlugInError(
+                    _("Unknown unit type: {!r}").format(unit_name)
+                )
+            try:
+                unit = unit_cls.from_rfc822_record(record, provider)
+            except ValueError as exc:
+                raise PlugInError(
+                    _("Cannot define unit from record {!r}: {}").format(
+                        record, exc
+                    )
+                )
+            if check:
+                for issue in unit.check(context=context, live=True):
+                    if issue.severity is Severity.error:
+                        raise PlugInError(
+                            _("Problem in unit definition, {}").format(issue)
+                        )
+            if validate:
+                try:
+                    unit.validate(**validation_kwargs)
+                except ValidationError as exc:
+                    raise PlugInError(
+                        _("Problem in unit definition, field {}: {}").format(
+                            exc.field, exc.problem
+                        )
+                    )
+            unit_list.append(unit)
+            logger.debug(_("Loaded %r"), unit)
+        return unit_list
+
+    def discover_units(
+        self,
+        inspect_result: "List[Unit]",
+        filename: str,
+        text: str,
+        provider: "Provider1",
+    ) -> "Iterable[Unit]":
+        for unit in inspect_result:
+            yield unit
+        yield self.make_file_unit(filename, provider)
+
+    # NOTE: this version of plugin_object() is just for legacy code support
+    @property
+    def plugin_object(self):
+        return self.unit_list
+
+    @staticmethod
+    def _get_unit_cls(unit_name):
+        """
+        Get a class that implements the specified unit
+        """
+        # TODO: transition to lazy plugin collection
+        all_units.load()
+        return all_units.get_by_name(unit_name).plugin_object
+
+
+
+
 
 class ProviderContentEnumerator:
     """
@@ -447,6 +539,7 @@ def _get_classify_fn_list(
             classify_fn_list.append(self._classify_pxu_jobs)
         if self.provider.units_dir:
             classify_fn_list.append(self._classify_pxu_units)
+            classify_fn_list.append(self._classify_yaml_units)
         if self.provider.data_dir:
             classify_fn_list.append(self._classify_data)
         if self.provider.bin_dir:
@@ -510,6 +603,18 @@ def _classify_pxu_units(self, filename: str):
                     self.provider.units_dir,
                     UnitPlugIn,
                 )
+    def _classify_yaml_units(self, filename: str):
+        """Classify yaml files in unit dirs as unit source."""
+        if filename.startswith(self.provider.units_dir):
+            ext = os.path.splitext(filename)[1]
+            if ext == ".yaml":
+                return (
+                    FileRole.unit_source,
+                    self.provider.units_dir,
+                    YamlUnitPlugIn,
+                )
+
+
 
     def _classify_data(self, filename: str):
         """classify files in data_dir as data"""

checkbox-ng/plainbox/impl/secure/yaml.py

@@ -0,0 +1,49 @@
+import yaml
+
+from plainbox.impl.secure.origin import Origin
+
+class LoaderWithMarks(yaml.Loader):
+    def construct_mapping(self, node, deep=False):
+        mapping = super().construct_mapping(node, deep=deep)
+        # attach line number for tracking the offset
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if isinstance(key, str):
+                mapping[key] = (self.construct_object(value_node, deep=deep),
+                                value_node.start_mark.line)
+        return mapping
+
+class YamlRecord:
+    """
+    Checkbox unit definitions encoded in YAML.
+    """
+
+    def __init__(self, data, origin=None, field_offset_map=None):
+        self.data = data
+        self.raw_data = data
+        self.field_offset_map = field_offset_map
+        if origin is None:
+            origin = Origin.get_caller_origin()
+        self.origin = origin
+
+    def dump(self):
+        return yaml.dump(self.data)
+
+def gen_yaml_records(stream, data_cls=dict, source=None):
+    if not isinstance(stream, str):
+        stream = open(stream.name).read()
+
+
+    records_with_mappings = yaml.load(stream, Loader=LoaderWithMarks)
+    for record in records_with_mappings:
+        data = dict()
+        field_offsets = dict()
+        for key, (value, offset) in record.items():
+            data[key] = value.strip()
+            field_offsets[key] = offset
+        origin = Origin(source, None, None)
+        yield YamlRecord(data, origin, field_offsets)
+
+def load_yaml_records(stream, data_cls=dict, source=None):
+    return list(gen_yaml_records(stream, data_cls, source))
+