From ce876fcea50ddf53b36d0c60e1298c761c515de0 Mon Sep 17 00:00:00 2001 From: Maciej Kisielewski Date: Thu, 2 May 2024 00:49:16 +0200 Subject: [PATCH] WIP yaml support --- .../plainbox/impl/secure/providers/v1.py | 105 ++++++++++++++++++ checkbox-ng/plainbox/impl/secure/yaml.py | 49 ++++++++ 2 files changed, 154 insertions(+) create mode 100644 checkbox-ng/plainbox/impl/secure/yaml.py diff --git a/checkbox-ng/plainbox/impl/secure/providers/v1.py b/checkbox-ng/plainbox/impl/secure/providers/v1.py index cb5fe84468..67b55191c8 100644 --- a/checkbox-ng/plainbox/impl/secure/providers/v1.py +++ b/checkbox-ng/plainbox/impl/secure/providers/v1.py @@ -27,6 +27,7 @@ import gettext import logging import os +import yaml from plainbox.abc import IProvider1 from plainbox.i18n import gettext as _ @@ -48,6 +49,7 @@ from plainbox.impl.secure.rfc822 import FileTextSource from plainbox.impl.secure.rfc822 import RFC822SyntaxError from plainbox.impl.secure.rfc822 import load_rfc822_records +from plainbox.impl.secure.yaml import load_yaml_records from plainbox.impl.unit import all_units from plainbox.impl.unit.file import FileRole from plainbox.impl.unit.file import FileUnit @@ -278,6 +280,96 @@ def _get_unit_cls(unit_name): all_units.load() return all_units.get_by_name(unit_name).plugin_object +class YamlUnitPlugIn(ProviderContentPlugIn): + """ + Loader of Checkbox units encoded in YAML. + """ + def inspect( + self, + filename: str, + text: str, + provider: "Provider1", + validate: bool, + validation_kwargs: "Dict[str, Any]", + check: bool, + context: "???", + ) -> "Any": + """ + Load all units from the YAML file. + """ + logger.debug(_("Loading units from %r..."), filename) + try: + records = load_yaml_records(text, source=FileTextSource(filename)) + except yaml.YAMLError as exc: + raise PlugInError( + _("Cannot load job definitions from {!r}: {}").format( + filename, exc + ) + ) + unit_list = [] + for record in records: + unit_name = record.data.get("unit", "job") + try: + unit_cls = self._get_unit_cls(unit_name) + except KeyError: + raise PlugInError( + _("Unknown unit type: {!r}").format(unit_name) + ) + try: + unit = unit_cls.from_rfc822_record(record, provider) + except ValueError as exc: + raise PlugInError( + _("Cannot define unit from record {!r}: {}").format( + record, exc + ) + ) + if check: + for issue in unit.check(context=context, live=True): + if issue.severity is Severity.error: + raise PlugInError( + _("Problem in unit definition, {}").format(issue) + ) + if validate: + try: + unit.validate(**validation_kwargs) + except ValidationError as exc: + raise PlugInError( + _("Problem in unit definition, field {}: {}").format( + exc.field, exc.problem + ) + ) + unit_list.append(unit) + logger.debug(_("Loaded %r"), unit) + return unit_list + + def discover_units( + self, + inspect_result: "List[Unit]", + filename: str, + text: str, + provider: "Provider1", + ) -> "Iterable[Unit]": + for unit in inspect_result: + yield unit + yield self.make_file_unit(filename, provider) + + # NOTE: this version of plugin_object() is just for legacy code support + @property + def plugin_object(self): + return self.unit_list + + @staticmethod + def _get_unit_cls(unit_name): + """ + Get a class that implements the specified unit + """ + # TODO: transition to lazy plugin collection + all_units.load() + return all_units.get_by_name(unit_name).plugin_object + + + + class ProviderContentEnumerator: """ @@ -447,6 +539,7 @@ def _get_classify_fn_list( classify_fn_list.append(self._classify_pxu_jobs) if self.provider.units_dir: classify_fn_list.append(self._classify_pxu_units) + classify_fn_list.append(self._classify_yaml_units) if self.provider.data_dir: classify_fn_list.append(self._classify_data) if self.provider.bin_dir: @@ -510,6 +603,18 @@ def _classify_pxu_units(self, filename: str): self.provider.units_dir, UnitPlugIn, ) + def _classify_yaml_units(self, filename: str): + """Classify yaml files in unit dirs as unit source.""" + if filename.startswith(self.provider.units_dir): + ext = os.path.splitext(filename)[1] + if ext == ".yaml": + return ( + FileRole.unit_source, + self.provider.units_dir, + YamlUnitPlugIn, + ) + + def _classify_data(self, filename: str): """classify files in data_dir as data""" diff --git a/checkbox-ng/plainbox/impl/secure/yaml.py b/checkbox-ng/plainbox/impl/secure/yaml.py new file mode 100644 index 0000000000..e6a519d48e --- /dev/null +++ b/checkbox-ng/plainbox/impl/secure/yaml.py @@ -0,0 +1,49 @@ +import yaml + +from plainbox.impl.secure.origin import Origin + +class LoaderWithMarks(yaml.Loader): + def construct_mapping(self, node, deep=False): + mapping = super().construct_mapping(node, deep=deep) + # attach line number for tracking the offset + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if isinstance(key, str): + mapping[key] = (self.construct_object(value_node, deep=deep), + value_node.start_mark.line) + return mapping + +class YamlRecord: + """ + Checkbox unit definitions encoded in YAML. + """ + + def __init__(self, data, origin=None, field_offset_map=None): + self.data = data + self.raw_data = data + self.field_offset_map = field_offset_map + if origin is None: + origin = Origin.get_caller_origin() + self.origin = origin + + def dump(self): + return yaml.dump(self.data) + +def gen_yaml_records(stream, data_cls=dict, source=None): + if not isinstance(stream, str): + stream = open(stream.name).read() + + + records_with_mappings = yaml.load(stream, Loader=LoaderWithMarks) + for record in records_with_mappings: + data = dict() + field_offsets = dict() + for key, (value, offset) in record.items(): + data[key] = value.strip() + field_offsets[key] = offset + origin = Origin(source, None, None) + yield YamlRecord(data, origin, field_offsets) + +def load_yaml_records(stream, data_cls=dict, source=None): + return list(gen_yaml_records(stream, data_cls, source)) +