Support rendering a simple recipe

pyproject.toml

@@ -27,16 +27,27 @@ classifiers = [
   "Typing :: Typed",
 ]
 dynamic = ["version"]
-dependencies = []
+dependencies = [
+  "click >=8",
+  "pyyaml",
+  "jinja2",
+]
 
 [project.optional-dependencies]
+# NOTE: "test" and "dev" are duplicated; why do we need both?
 test = [
+  "mypy",
   "pytest >=6",
   "pytest-cov >=3",
+  "types-click",
+  "types-pyyaml",
 ]
 dev = [
+  "mypy",
   "pytest >=6",
   "pytest-cov >=3",
+  "types-click",
+  "types-pyyaml",
 ]
 docs = [
   "sphinx>=7.0",
@@ -53,6 +64,10 @@ Discussions = "https://github.com/qgreenland-net/ogdc-runner/discussions"
 Changelog = "https://github.com/qgreenland-net/ogdc-runner/releases"
 
 
+[project.scripts]
+ogdc-runner = "ogdc_runner.__main__:cli"
+
+
 [build-system]
 requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
@@ -102,6 +117,12 @@ module = "ogdc_runner.*"
 disallow_untyped_defs = true
 disallow_incomplete_defs = true
 
+[[tool.mypy.overrides]]
+module = [
+  "jinja2.*",
+]
+ignore_missing_imports = true
+
 
 [tool.ruff]
 src = ["src"]

src/ogdc_runner/__main__.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+
+from ogdc_runner.recipe.simple import render_simple_recipe
+
+# TODO: How do we handle e.g. GitHub URL to recipe?
+recipe_path = click.argument(
+    "recipe_path",
+    required=True,
+    metavar="PATH",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+        readable=True,
+        resolve_path=True,
+        path_type=Path,
+    ),
+)
+
+
+@click.group
+def cli() -> None:
+    """A tool for submitting data transformation recipes to OGDC for execution."""
+    pass
+
+
+@cli.command
+@recipe_path
+def render(recipe_path: Path) -> None:
+    """Render a recipe, but don't submit it.
+
+    Useful for testing.
+    """
+    render_simple_recipe(recipe_path)
+
+
+@cli.command
+@recipe_path
+def submit(recipe_path: Path) -> None:
+    """Render and submit a recipe to OGDC for execution."""
+    raise NotImplementedError("Not yet!")

src/ogdc_runner/constants.py

@@ -0,0 +1,6 @@
+# .txt? To make sure it's clear the file can't be executed directly
+from __future__ import annotations
+
+SIMPLE_RECIPE_FILENAME = "recipe.sh"
+# .yaml and .yml?
+RECIPE_CONFIG_FILENAME = "meta.yml"

src/ogdc_runner/recipe/__init__.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from ogdc_runner.constants import RECIPE_CONFIG_FILENAME
+
+
+def get_recipe_config(recipe_directory: Path) -> dict:
+    """Extract config from a recipe configuration file (meta.yml)."""
+    with open(recipe_directory / RECIPE_CONFIG_FILENAME) as config_file:
+        return yaml.safe_load(config_file)

src/ogdc_runner/recipe/simple.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from jinja2 import Environment, PackageLoader
+
+from ogdc_runner.constants import SIMPLE_RECIPE_FILENAME
+from ogdc_runner.recipe import get_recipe_config
+
+environment = Environment(loader=PackageLoader("ogdc_runner"))
+template = environment.get_template("simple_recipe.py.j2")
+
+input_subkeys = {
+    "url": "beam.io.SomeTransformThatCanReadAFileFromAUrl",
+    "dataone_doi": "our_custom_transforms.DataOneDoiInput",
+}
+
+
+def _get_commands(simple_recipe_path: Path) -> list[str]:
+    """Extract commands from a simple recipe file."""
+    # read_lines is going to be more efficient I assume...
+    lines = simple_recipe_path.read_text().split("\n")
+
+    # Omit comments and empty lines
+    commands = [line for line in lines if line and not line.startswith("#")]
+    return commands
+
+
+def _get_input_constructor_and_arg(config: dict) -> tuple[type, any]:
+    acceptable_values = f"Acceptable values: {input_subkeys.keys()}"
+    if num_keys := len(config["input"].keys()) > 1:
+        raise RuntimeError(
+            f"Expected 1 sub-key for the `input` key; got {num_keys}."
+            f" {acceptable_values}"
+        )
+
+    key, val = list(config["input"].items())[0]
+
+    try:
+        clss = input_subkeys[key]
+    except KeyError:
+        raise RuntimeError(
+            f"Received unexecpected sub-key for `input` key: {key}"
+            f" {acceptable_values}"
+        )
+
+    return clss, val
+
+
+def render_simple_recipe(recipe_directory: Path):
+    commands = _get_commands(recipe_directory / SIMPLE_RECIPE_FILENAME)
+    config = get_recipe_config(recipe_directory)
+
+    input_constructor, input_constructor_arg = _get_input_constructor_and_arg(config)
+
+    print(
+        template.render(
+            commands=commands,
+            input_constructor=input_constructor,
+            input_constructor_arg=input_constructor_arg,
+        )
+    )

src/ogdc_runner/templates/simple_recipe.py.j2

@@ -0,0 +1,6 @@
+recipe = (
+  {{input_constructor}}({{input_constructor_arg}})
+{%- for cmd in commands %}
+  | CommandPTransform("""{{cmd}}""")
+{%- endfor %}
+)