Merge pull request #41 from QGreenland-Net/issue-40-links

Issue 40 - allow links

README.md

@@ -57,7 +57,7 @@ ogdc-runner submit /path/to/ogdc-recipes/my-recipe
 Or:
 
 ```bash
-ogdc-runner submit https://github.com/QGreenland-Net/ogdc-recipes/ my-recipe
+ogdc-runner submit github://qgreenland-net:ogdc-recipes@main/recipes/seal-tags
 ```
 
 ## Implementation notes

docs/CONTRIBUTING.md

@@ -5,6 +5,8 @@ description of best practices for developing scientific packages.
 
 [spc-dev-intro]: https://learn.scientific-python.org/development/
 
+> [!NOTE] Add yourself as an author in [pyproject.toml](../pyproject.toml)
+
 ## Setting up a development environment manually
 
 First, ensure you have [ogdc-helm](https://github.com/QGreenland-Net/ogdc-helm)

pyproject.toml

@@ -4,6 +4,7 @@ authors = [
   { name = "Matt Fisher", email = "[email protected]" },
   { name = "Rushiraj Nenuji", email = "[email protected]" },
   { name = "Trey Stafford", email = "[email protected]" },
+  { name = "Robyn Marowitz", email = "[email protected]" },
 ]
 description = "Defines OGDC recipe API(s) and submits recipes to OGDC for execution"
 readme = "README.md"
@@ -29,6 +30,7 @@ dependencies = [
   "hera >=5.18.0",
   "pydantic >=2.7.1",
   "pyyaml",
+  "fsspec",
 ]
 
 [project.optional-dependencies]
@@ -118,6 +120,7 @@ disallow_incomplete_defs = true
 module = [
   "jinja2.*",
   "kubernetes.*",
+  "fsspec",
 ]
 ignore_missing_imports = true
 

src/ogdc_runner/__main__.py

@@ -1,26 +1,17 @@
 from __future__ import annotations
 
 import time
-from pathlib import Path
 
 import click
 
 from ogdc_runner.argo import get_workflow_status, submit_workflow
 from ogdc_runner.recipe.simple import make_simple_workflow
 
-# TODO: How do we handle e.g. GitHub URL to recipe?
 recipe_path = click.argument(
     "recipe_path",
     required=True,
-    metavar="PATH",
-    type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        readable=True,
-        resolve_path=True,
-        path_type=Path,
-    ),
+    metavar="RECIPE-PATH",
+    type=str,
 )
 
 
@@ -29,7 +20,7 @@ def cli() -> None:
     """A tool for submitting data transformation recipes to OGDC for execution."""
 
 
-def _submit_workflow(recipe_path: Path) -> str:
+def _submit_workflow(recipe_path: str) -> str:
     workflow = make_simple_workflow(
         recipe_dir=recipe_path,
     )
@@ -40,8 +31,13 @@ def _submit_workflow(recipe_path: Path) -> str:
 
 @cli.command
 @recipe_path
-def submit(recipe_path: Path) -> None:
-    """Submit a recipe to OGDC for execution."""
+def submit(recipe_path: str) -> None:
+    """
+    Submit a recipe to OGDC for execution.
+
+    RECIPE-PATH: Path to the recipe file. Use either a local path (e.g., '/ogdc-recipes/recipes/seal-tags')
+    or an fsspec-compatible GitHub string (e.g., 'github://qgreenland-net:ogdc-recipes@main/recipes/seal-tags').
+    """
     _submit_workflow(recipe_path)
 
 
@@ -59,8 +55,13 @@ def check_workflow_status(workflow_name: str) -> None:
 
 @cli.command
 @recipe_path
-def submit_and_wait(recipe_path: Path) -> None:
-    """Submit a recipe to OGDC for execution and wait until completion."""
+def submit_and_wait(recipe_path: str) -> None:
+    """
+    Submit a recipe to OGDC for execution and wait until completion.
+
+    RECIPE-PATH: Path to the recipe file. Use either a local path (e.g., '/ogdc-recipes/recipes/seal-tags')
+    or an fsspec-compatible GitHub string (e.g., 'github://qgreenland-net:ogdc-recipes@main/recipes/seal-tags').
+    """
     workflow_name = _submit_workflow(recipe_path)
 
     while True:

src/ogdc_runner/recipe/__init__.py

@@ -1,16 +1,16 @@
 from __future__ import annotations
 
-from pathlib import Path
-
+import fsspec
 import yaml
 
 from ogdc_runner.constants import RECIPE_CONFIG_FILENAME
 from ogdc_runner.models.recipe_config import RecipeConfig
 
 
-def get_recipe_config(recipe_directory: Path) -> RecipeConfig:
+def get_recipe_config(recipe_directory: str) -> RecipeConfig:
     """Extract config from a recipe configuration file (meta.yml)."""
-    with (recipe_directory / RECIPE_CONFIG_FILENAME).open() as config_file:
+    recipe_path = f"{recipe_directory}/{RECIPE_CONFIG_FILENAME}"
+    with fsspec.open(recipe_path, "rt") as config_file:
         config_dict = yaml.safe_load(config_file)
 
     config = RecipeConfig(**config_dict)

src/ogdc_runner/recipe/simple.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-from pathlib import Path
-
+import fsspec
 from hera.workflows import (
     Artifact,
     Container,
@@ -46,7 +45,7 @@ def _make_fetch_url_template(recipe_config: RecipeConfig) -> Container:
     return template
 
 
-def _cmds_from_simple_recipe(recipe_dir: Path) -> list[str]:
+def _cmds_from_simple_recipe(recipe_dir: str) -> list[str]:
     """Read commands from a 'simple' OGDC recipe.
 
     'simple' OGDC recipes are `.sh` files containing bash commands. Commands can
@@ -55,16 +54,17 @@ def _cmds_from_simple_recipe(recipe_dir: Path) -> list[str]:
         * `/output_dir/`: output written by each command. It is expected that
           each command in a simple recipe will place data in `/output_dir/`.
     """
-    recipe_path = recipe_dir / SIMPLE_RECIPE_FILENAME
-    # read the commands from the recipe
-    lines = recipe_path.read_text().split("\n")
-    # Filter out comments. We assume all other lines are bash commands.
+    recipe_path = f"{recipe_dir}/{SIMPLE_RECIPE_FILENAME}"
+    print(f"Reading recipe from {recipe_path}")
+
+    with fsspec.open(recipe_path, "rt") as f:
+        lines = f.read().split("\n")
     commands = [line for line in lines if line and not line.startswith("#")]
 
     return commands
 
 
-def make_simple_workflow(recipe_dir: Path) -> Workflow:
+def make_simple_workflow(recipe_dir: str) -> Workflow:
     """Run the workflow and return its name as a str."""
     commands = _cmds_from_simple_recipe(recipe_dir)
     recipe_config = get_recipe_config(recipe_dir)

tests/integration/test_argo_workflow.py

@@ -11,7 +11,7 @@
 from ogdc_runner.argo import ARGO_WORKFLOW_SERVICE
 from ogdc_runner.recipe.simple import make_simple_workflow
 
-SIMPLE_RECIPE_TEST_PATH = Path(__file__).parent / "test_recipe"
+SIMPLE_RECIPE_TEST_PATH = str(Path(__file__).parent / "test_recipe")
 
 
 def test_simple_argo_workflow():