Implement initial data fetch from errata (#12)

Signed-off-by: Ondrej Moris <[email protected]>

.pre-commit-config.yaml

@@ -24,6 +24,10 @@ repos:
           - "attrs>=20.3.0"
           - "ruamel.yaml>=0.16.6"
           - "jinja2>=2.11.3"
+          # prevent conflict between types-requests and urllib3
+          - "types-requests<2.31.0.7; python_version < '3.10'"
+          - "types-requests; python_version >= '3.10'"
+
         pass_filenames: false
         args: [--config-file=pyproject.toml]
 

newa/__init__.py

@@ -1,14 +1,20 @@
+from __future__ import annotations
+
 import io
+import os
+import time
 from enum import Enum
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional, TypeVar
 
 import attrs
 import jinja2
+import requests
 import ruamel.yaml
 import ruamel.yaml.nodes
 import ruamel.yaml.representer
-from attrs import define, field
+from attrs import define, field, frozen, validators
+from requests_kerberos import HTTPKerberosAuth
 
 if TYPE_CHECKING:
     from typing_extensions import Self, TypeAlias
@@ -82,6 +88,19 @@ def render_template(
         raise Exception("Could not render template.") from exc
 
 
+def krb_get_request(url: str, attempts: int = 5, delay: int = 5) -> Any:
+    """ Generic GET request using Kerberos authentication """
+
+    while attempts:
+        r = requests.get(url, auth=HTTPKerberosAuth(delegate=True))
+        if r.status_code == 200:
+            return r.json()
+        time.sleep(delay)
+        attempts -= 1
+
+    raise Exception(f"GET request to {url} failed")
+
+
 class EventType(Enum):
     """ Event types """
 
@@ -92,7 +111,7 @@ class EventType(Enum):
 class Cloneable:
     """ A class whose instances can be cloned """
 
-    def clone(self) -> 'Self':
+    def clone(self) -> Self:
         return attrs.evolve(self)
 
 
@@ -126,7 +145,33 @@ class Event(Serializable):
     """ A triggering event of Newa pipeline """
 
     type_: EventType = field(converter=EventType)
-    id: 'ErratumId'
+    id: ErratumId
+
+
+@frozen
+class ErrataTool:
+    """
+    Interface to Errata Tool instance
+
+    Its only purpose is to serve information about an erratum and its builds.
+    """
+
+    # TODO: Until we have a dedicated newa config, we'll setup ET instance
+    # url via environment variable NEWA_ET_URL
+    url: str = field(validator=validators.matches_re("^https?://.+$"))
+
+    @url.default  # pyright: ignore [reportAttributeAccessIssue]
+    def _url_factory(self) -> str:
+        if "NEWA_ET_URL" in os.environ:
+            return os.environ["NEWA_ET_URL"]
+        raise Exception("NEWA_ET_URL envvar is required.")
+
+    # TODO: Not used at this point because we only consume builds now
+    def fetch_info(self, erratum_id: str) -> Any:
+        return krb_get_request(f"{self.url}/advisory/{erratum_id}.json")
+
+    def fetch_releases(self, erratum_id: str) -> Any:
+        return krb_get_request(f"{self.url}/advisory/{erratum_id}/builds.json")
 
 
 @define
@@ -145,13 +190,63 @@ class InitialErratum(Serializable):
 
 @define
 class Erratum(Cloneable, Serializable):
-    """ An eratum """
+    """
+    An eratum
 
+    Represents a set of builds targetting a single release.
+    """
+
+    # TODO: We might need to add more in the future.
     release: str
     builds: list[str] = field(factory=list)
 
-    def fetch_details(self) -> None:
-        raise NotImplementedError
+    @classmethod
+    def from_errata_tool(cls, event: Event) -> list[Erratum]:
+        """
+        Creates a list of Erratum instances based on given errata ID
+
+        Errata is split into one or more instances of an erratum. There is one
+        for each release included in errata. Each errata has a single release
+        set - it is either regular one or ASYNC. An errata with a regular
+        release (e.g. RHEL-9.0.0.Z.EUS) will result into a single erratatum.
+        On the other hand an errata with ASYNC release might result into one
+        or more instances of erratum.
+        """
+
+        errata = []
+
+        # TODO: We might need to assert QE (or later) state. There is no point
+        # in fetching errata in NEW_FILES where builds need not to be present.
+
+        # In QE state there is are zero or more builds in an erratum, each
+        # contains one or more packages, e.g.:
+        # {
+        #   "RHEL-9.0.0.Z.EUS": [
+        #     {
+        #       "scap-security-guide-0.1.72-1.el9_3": {
+        #          ...
+        #     }
+        #   ]
+        #   "RHEL-9.2.0.Z.EUS": [
+        #     {
+        #       "scap-security-guide-0.1.72-1.el9_3": {
+        #          ...
+        #     }
+        #   ]
+        # }
+
+        releases_json = ErrataTool().fetch_releases(event.id)
+        for release in releases_json:
+            builds = []
+            builds_json = releases_json[release]
+            for item in builds_json:
+                builds += list(item.keys())
+            if builds:
+                errata.append(cls(release, builds))
+            else:
+                raise Exception(f"No builds found in ER#{event.id}")
+
+        return errata
 
 
 @define
@@ -170,9 +265,6 @@ class Recipe(Cloneable, Serializable):
 
     url: str
 
-    def fetch_details(self) -> None:
-        raise NotImplementedError
-
 
 @define
 class Job(Cloneable, Serializable):

newa/cli.py

@@ -112,30 +112,27 @@ def main(click_context: click.Context, state_dir: str) -> None:
     multiple=True,
     )
 @click.pass_obj
-def cmd_event(ctx: CLIContext, errata_ids: tuple[str, ...]) -> None:
+def cmd_event(ctx: CLIContext, errata_ids: list[str]) -> None:
     ctx.enter_command('event')
 
-    if errata_ids:
-        for erratum_id in errata_ids:
-            event = Event(type_=EventType.ERRATUM, id=erratum_id)
+    # Errata IDs were not given, try to load them from init- files.
+    if not errata_ids:
+        errata_ids = [e.event.id for e in ctx.load_initial_errata('init-')]
 
-            # fetch erratum details, namely releases
-            releases = ['RHEL-8.10.0', 'RHEL-9.4.0']
+    # Abort if there are still no errata IDs.
+    if not errata_ids:
+        raise Exception('Missing errata IDs!')
 
-            for release in releases:
-                erratum_job = ErratumJob(event=event, erratum=Erratum(release=release))
+    for erratum_id in errata_ids:
+        event = Event(type_=EventType.ERRATUM, id=erratum_id)
 
-                ctx.save_erratum_job('event-', erratum_job)
+        # fetch erratum details (TODO: releases for now)
+        errata = Erratum.from_errata_tool(event)
 
-    else:
-        for erratum in ctx.load_initial_errata('init-'):
-            # fetch erratum details, namely releases
-            releases = ['RHEL-8.10.0', 'RHEL-9.4.0']
+        for erratum in errata:
+            erratum_job = ErratumJob(event=event, erratum=erratum)
 
-            for release in releases:
-                erratum_job = ErratumJob(event=erratum.event, erratum=Erratum(release=release))
-
-                ctx.save_erratum_job('event-', erratum_job)
+            ctx.save_erratum_job('event-', erratum_job)
 
 
 @main.command(name='jira')

pyproject.toml

@@ -75,6 +75,9 @@ dependencies = [
     "mypy",
     "pytest",
     "pre-commit",
+    # prevent conflict between types-requests and urllib3
+    "types-requests<2.31.0.7; python_version < '3.10'",
+    "types-requests; python_version >= '3.10'",
     ]
 
 [tool.hatch.envs.dev.scripts]
@@ -117,7 +120,9 @@ python_version = "3.9"
 files = ["newa/"]
 
 [[tool.mypy.overrides]]
-module = []
+module = [
+    "requests_kerberos.*"
+]
 ignore_missing_imports = true
 
 [tool.autopep8]
@@ -163,6 +168,7 @@ select = [
     "RUF",  # ruff
     ]
 ignore = [
+    "UP007",  # Use `X | Y` for type annotations
 ]
 
 [tool.ruff.isort]