Add minimal RequestJob code

This PR adds minimal code for 'schedule' subcommand, implementing
associated classes Request and RequestJob.

README.md

@@ -12,3 +12,120 @@ $ newa
 Newa!
 $
 ```
+
+## Architecture
+
+### Subcommand `event`
+
+Gets event details either from a command line
+```
+newa event --errata 12345
+```
+or from a files having `init-` prefix.
+
+Produces multiple files based on the event (erratum) details,
+splitting them according to the product release and populating
+them with the `event` and `erratum` keys.
+
+For example:
+```
+$ cat state/event-128049-RHEL-9.4.0.yaml
+erratum:
+  builds: []
+  release: RHEL-9.4.0
+event:
+  id: '128049'
+  type_: erratum
+
+$ cat state/event-128049-RHEL-8.10.0.yaml
+erratum:
+  builds: []
+  release: RHEL-8.10.0
+event:
+  id: '128049'
+  type_: erratum
+
+```
+
+### Subcommand `jira`
+
+Processes multiple files having `event-` prefix. For each event/file reads
+Jira configuration file and for each item from the configuration it
+creates or updates a Jira issue and produces `jira-` file, populating it
+with `jira` and `recipe` keys.
+
+For example:
+```
+$ cat state/jira-128049-RHEL-8.10.0-NEWA-12.yaml
+erratum:
+  builds: []
+  release: RHEL-8.10.0
+event:
+  id: '128049'
+  type_: erratum
+jira:
+  id: NEWA-12
+recipe:
+  url: https://path/to/recipe.yaml
+
+$ cat state/jira-128049-RHEL-9.4.0-NEWA-6.yaml
+erratum:
+  builds: []
+  release: RHEL-9.4.0
+event:
+  id: '128049'
+  type_: erratum
+jira:
+  id: NEWA-6
+recipe:
+  url: https://path/to/recipe.yaml
+```
+
+### Subcommand `schedule`
+
+Processes multiple files having `jira-` prefix. For each such file it
+reads recipe details from `recipe.url` and according to that recipe
+it produces multiple `request-` files, populating it with `recipe` key.
+
+For example:
+```
+$ cat state/request-128049-RHEL-8.10.0-NEWA-12-REQ-1.yaml
+erratum:
+  builds: []
+  release: RHEL-8.10.0
+event:
+  id: '128049'
+  type_: erratum
+jira:
+  id: NEWA-12
+recipe:
+  url: https://path/to/recipe.yaml
+request:
+  context:
+    distro: rhel-8.10.0
+  environment: {}
+  git_ref: ''
+  git_url: ''
+  id: REQ-1
+  tmt_path: ''
+
+$ cat state/request-128049-RHEL-8.10.0-NEWA-12-REQ-2.yaml
+erratum:
+  builds: []
+  release: RHEL-8.10.0
+event:
+  id: '128049'
+  type_: erratum
+jira:
+  id: NEWA-12
+recipe:
+  url: https://path/to/recipe.yaml
+request:
+  context:
+    distro: rhel-8.10.0
+  environment: {}
+  git_ref: ''
+  git_url: ''
+  id: REQ-2
+  tmt_path: ''
+```

component-config.yaml.sample

@@ -46,4 +46,4 @@ issues:
    id: subtask_regression
    parent_id: errata_task
    on_respin: close
-   job_recipe: https://path/to/recipe.yaml
+   job_recipe: https://raw.githubusercontent.com/RedHatQE/newa/ks_recipe_job/component-recipe.yaml.sample

component-recipe.yaml.sample

@@ -0,0 +1,22 @@
+fixtures:
+    git_url: https://github.com/RedHatQE/newa.git
+    git_ref: main
+    context:
+        tier: 1
+    environment:
+        DESCRIPTION: "This is a test recipe"
+dimensions:
+    arch:
+       - context:
+             arch: x86_64
+       - context:
+             arch: aarch64
+    distro:
+       - context:
+             fips: yes
+         environment:
+             FIPS: "FIPS ENABLED"
+       - context:
+             fips: no
+         environment:
+             FIPS: "FIPS NOT ENABLED"

newa/__init__.py

@@ -1,10 +1,13 @@
+import copy
 import io
+import itertools
 from enum import Enum
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional, TypeVar
 
 import attrs
 import jinja2
+import requests
 import ruamel.yaml
 import ruamel.yaml.nodes
 import ruamel.yaml.representer
@@ -120,6 +123,13 @@ def from_yaml(cls: type[SerializableT], serialized: str) -> SerializableT:
     def from_yaml_file(cls: type[SerializableT], filepath: Path) -> SerializableT:
         return cls.from_yaml(filepath.read_text())
 
+    @classmethod
+    def from_yaml_url(cls: type[SerializableT], url: str) -> SerializableT:
+        r = requests.get(url)
+        if r.status_code == 200:
+            return cls.from_yaml(r.text)
+        raise Exception(f"GET request to {url} failed")
+
 
 @define
 class Event(Serializable):
@@ -174,6 +184,62 @@ def fetch_details(self) -> None:
         raise NotImplementedError
 
 
+@define
+class RecipeConfig(Cloneable, Serializable):
+    """ A job recipe configuration """
+
+    fixtures: dict = {}
+    dimensions: dict = {}
+
+    def build_requests(self) -> list:
+
+        # this is here to generate unique recipe IDs
+        recipe_id_gen = itertools.count(start=1)
+
+        # get all options from dimentions
+        options = []
+        for dimension in self.dimensions:
+            options.append(self.dimensions[dimension])
+        # generate combinations
+        combinations = list(itertools.product(*options))
+        # extend each combination with fixtures
+        for i in range(len(combinations)):
+            combinations[i] = (self.fixtures,) + (combinations[i])
+
+        def merge_combination_data(combination):
+            merged = {}
+            for record in combination:
+                for key in record:
+                    if key not in merged:
+                        # we need to do a deep copy so we won't corrupt the original data
+                        merged[key] = copy.deepcopy(record[key])
+                    elif isinstance(merged[key], dict) and isinstance(record[key], dict):
+                        merged[key].update(record[key])
+                    else:
+                        raise Exception(f"Don't know how to merge record type {key}")
+            return merged
+
+        # now for each combination merge data from individual dimensions
+        merged_combinations = list(map(merge_combination_data, combinations))
+        return [Request(id=f'REQ-{next(recipe_id_gen)}', **combination)
+                for combination in merged_combinations]
+
+
+@define
+class Request(Cloneable, Serializable):
+    """ A test job request configuration """
+
+    id: str
+    context: Optional[dict] = {}
+    environment: Optional[dict] = {}
+    git_url: Optional[str] = ''
+    git_ref: Optional[str] = ''
+    tmt_path: Optional[str] = ''
+
+    def fetch_details(self) -> None:
+        raise NotImplementedError
+
+
 @define
 class Job(Cloneable, Serializable):
     """ A single job """
@@ -202,7 +268,7 @@ class ErratumJob(Job):
 
     @property
     def id(self) -> str:
-        return f'{self.event.id} @ {self.erratum.release}'
+        return f'E: {self.event.id} @ {self.erratum.release}'
 
 
 @define
@@ -219,7 +285,20 @@ class JiraJob(ErratumJob):
 
     @property
     def id(self) -> str:
-        return f'{self.event.id} @ {self.erratum.release}'
+        return f'J: {self.event.id} @ {self.erratum.release} - {self.jira.id}'
+
+
+@define
+class RequestJob(JiraJob):
+    """ A single *request* to be scheduled for execution """
+
+    request = field(  # type: ignore[var-annotated]
+        converter=lambda x: x if isinstance(x, Request) else Request(**x),
+        )
+
+    @property
+    def id(self) -> str:
+        return f'R: {self.event.id} @ {self.erratum.release} - {self.jira.id} / {self.request.id}'
 
 
 #

newa/cli.py

@@ -18,6 +18,8 @@
     Issue,
     JiraJob,
     Recipe,
+    RecipeConfig,
+    RequestJob,
     render_template,
     )
 
@@ -69,6 +71,20 @@ def load_erratum_jobs(self, filename_prefix: str) -> Iterator[ErratumJob]:
 
             yield self.load_erratum_job(self.state_dirpath / child)
 
+    def load_jira_job(self, filepath: Path) -> JiraJob:
+        job = JiraJob.from_yaml_file(filepath)
+
+        self.logger.info(f'Discovered jira job {job.id} in {filepath}')
+
+        return job
+
+    def load_jira_jobs(self, filename_prefix: str) -> Iterator[JiraJob]:
+        for child in self.state_dirpath.iterdir():
+            if not child.name.startswith(filename_prefix):
+                continue
+
+            yield self.load_jira_job(self.state_dirpath / child)
+
     def save_erratum_job(self, filename_prefix: str, job: ErratumJob) -> None:
         filepath = self.state_dirpath / \
             f'{filename_prefix}{job.event.id}-{job.erratum.release}.yaml'
@@ -87,6 +103,13 @@ def save_jira_job(self, filename_prefix: str, job: JiraJob) -> None:
         job.to_yaml_file(filepath)
         self.logger.info(f'Jira job {job.id} written to {filepath}')
 
+    def save_request_job(self, filename_prefix: str, job: RequestJob) -> None:
+        filepath = self.state_dirpath / \
+            f'{filename_prefix}{job.event.id}-{job.erratum.release}-{job.jira.id}-{job.request.id}.yaml'
+
+        job.to_yaml_file(filepath)
+        self.logger.info(f'Request job {job.id} written to {filepath}')
+
 
 @click.group(chain=True)
 @click.option(
@@ -206,12 +229,24 @@ def cmd_jira(ctx: CLIContext) -> None:
 def cmd_schedule(ctx: CLIContext) -> None:
     ctx.enter_command('schedule')
 
-    for erratum_job in ctx.load_erratum_jobs('jira-'):
-        # prepare  parameters based on errata details (environment variables)
-        # generate all relevant test jobs using the recipe
-        # prepares a list of JobExec objects
-
-        ctx.save_erratum_job('schedule-', erratum_job)
+    for jira_job in ctx.load_jira_jobs('jira-'):
+        # prepare parameters based on the recipe from recipe.url
+        # generate all relevant test request using the recipe data
+        # prepare a list of Request objects
+
+        config = RecipeConfig.from_yaml_url(jira_job.recipe.url)
+        print(config)
+        requests = config.build_requests()
+
+        # create few fake Issue objects for now
+        for request in requests:
+            request_job = RequestJob(
+                event=jira_job.event,
+                erratum=jira_job.erratum,
+                jira=jira_job.jira,
+                recipe=jira_job.recipe,
+                request=request)
+            ctx.save_request_job('request-', request_job)
 
 
 @main.command(name='execute')