feat: recipe modifications (update_version, update_build_number) (#47)

pixi.toml

@@ -34,7 +34,6 @@ pre-commit = ">=3.7.1,<4"
 pre-commit-hooks = ">=4.6.0,<5"
 typos = ">=1.23.1,<2"
 mypy = ">=1.10.1,<2"
-types-pyyaml = ">=6.0.12.20240311,<6.0.13"
 ruff = ">=0.5.0,<0.6"
 
 [feature.lint.tasks]
@@ -43,6 +42,8 @@ pre-commit-run = "pre-commit run"
 
 [feature.type-checking.dependencies]
 mypy = ">=1.10.1,<2"
+types-requests = ">=2.32.0.20240712,<3"
+types-pyyaml = ">=6.0.12.20240311,<6.0.13"
 
 [feature.type-checking.tasks]
 type-check = "mypy src"

src/rattler_build_conda_compat/jinja/jinja.py

@@ -107,11 +107,11 @@ def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, A
     """
     env = jinja_env()
     context = recipe_content.get("context", {})
-    # load all context templates
-    context_templates = load_recipe_context(context, env)
+    # render out the context section and retrieve dictionary
+    context_variables = load_recipe_context(context, env)
 
     # render the rest of the document with the values from the context
     # and keep undefined expressions _as is_.
     template = env.from_string(yaml.dump(recipe_content))
-    rendered_content = template.render(context_templates)
+    rendered_content = template.render(context_variables)
     return load_yaml(rendered_content)

src/rattler_build_conda_compat/modify_recipe.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import io
+import logging
+import re
+from typing import TYPE_CHECKING, Any, Literal
+
+import requests
+from ruamel.yaml import YAML
+
+from rattler_build_conda_compat.jinja.jinja import jinja_env, load_recipe_context
+from rattler_build_conda_compat.recipe_sources import Source, get_all_sources
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+yaml = YAML()
+yaml.preserve_quotes = True
+yaml.width = 4096
+yaml.indent(mapping=2, sequence=4, offset=2)
+
+
+def _update_build_number_in_context(recipe: dict[str, Any], new_build_number: int) -> bool:
+    for key in recipe.get("context", {}):
+        if key.startswith("build_") or key == "build":
+            recipe["context"][key] = new_build_number
+            return True
+    return False
+
+
+def _update_build_number_in_recipe(recipe: dict[str, Any], new_build_number: int) -> bool:
+    is_modified = False
+    if "build" in recipe and "number" in recipe["build"]:
+        recipe["build"]["number"] = new_build_number
+        is_modified = True
+
+    if "outputs" in recipe:
+        for output in recipe["outputs"]:
+            if "build" in output and "number" in output["build"]:
+                output["build"]["number"] = new_build_number
+                is_modified = True
+
+    return is_modified
+
+
+def update_build_number(file: Path, new_build_number: int = 0) -> str:
+    """
+    Update the build number in the recipe file.
+
+    Arguments:
+    ----------
+    * `file` - The path to the recipe file.
+    * `new_build_number` - The new build number to use. (default: 0)
+
+    Returns:
+    --------
+    The updated recipe as a string.
+    """
+    with file.open("r") as f:
+        data = yaml.load(f)
+    build_number_modified = _update_build_number_in_context(data, new_build_number)
+    if not build_number_modified:
+        _update_build_number_in_recipe(data, new_build_number)
+
+    with io.StringIO() as f:
+        yaml.dump(data, f)
+        return f.getvalue()
+
+
+class CouldNotUpdateVersionError(Exception):
+    NO_CONTEXT = "Could not find context in recipe"
+    NO_VERSION = "Could not find version in recipe context"
+
+    def __init__(self, message: str = "Could not update version") -> None:
+        self.message = message
+        super().__init__(self.message)
+
+
+class Hash:
+    def __init__(self, hash_type: Literal["md5", "sha256"], hash_value: str) -> None:
+        self.hash_type = hash_type
+        self.hash_value = hash_value
+
+    def __str__(self) -> str:
+        return f"{self.hash_type}: {self.hash_value}"
+
+
+def _has_jinja_version(url: str) -> bool:
+    """Check if the URL has a jinja `${{ version }}` in it."""
+    pattern = r"\${{\s*version"
+    return re.search(pattern, url) is not None
+
+
+def update_hash(source: Source, url: str, hash_: Hash | None) -> None:
+    """
+    Update the sha256 hash in the source dictionary.
+
+    Arguments:
+    ----------
+    * `source` - The source dictionary to update.
+    * `url` - The URL to download and hash (if no hash is provided).
+    * `hash_` - The hash to use. If not provided, the file will be downloaded and `sha256` hashed.
+    """
+    if "md5" in source:
+        del source["md5"]
+    if "sha256" in source:
+        del source["sha256"]
+
+    if hash_ is not None:
+        source[hash_.hash_type] = hash_.hash_value
+    else:
+        # download and hash the file
+        hasher = hashlib.sha256()
+        logger.info("Retrieving and hashing %s", url)
+        with requests.get(url, stream=True, timeout=100) as r:
+            for chunk in r.iter_content(chunk_size=4096):
+                hasher.update(chunk)
+        source["sha256"] = hasher.hexdigest()
+
+
+def update_version(file: Path, new_version: str, hash_: Hash | None) -> str:
+    """
+    Update the version in the recipe file.
+
+    Arguments:
+    ----------
+    * `file` - The path to the recipe file.
+    * `new_version` - The new version to use.
+    * `hash_type` - The hash type to use. If not provided, the file will be downloaded and `sha256` hashed.
+
+    Returns:
+    --------
+    The updated recipe as a string.
+    """
+
+    with file.open("r") as f:
+        data = yaml.load(f)
+
+    if "context" not in data:
+        raise CouldNotUpdateVersionError(CouldNotUpdateVersionError.NO_CONTEXT)
+    if "version" not in data["context"]:
+        raise CouldNotUpdateVersionError(CouldNotUpdateVersionError.NO_VERSION)
+
+    data["context"]["version"] = new_version
+
+    # set up the jinja context
+    env = jinja_env()
+    context = copy.deepcopy(data.get("context", {}))
+    context_variables = load_recipe_context(context, env)
+    # for r-recipes we add the default `cran_mirror` variable
+    context_variables["cran_mirror"] = "https://cran.r-project.org"
+
+    for source in get_all_sources(data):
+        # render the whole URL and find the hash
+        if "url" not in source:
+            continue
+
+        url = source["url"]
+        if isinstance(url, list):
+            url = url[0]
+
+        if not _has_jinja_version(url):
+            continue
+
+        template = env.from_string(url)
+        rendered_url = template.render(context_variables)
+
+        update_hash(source, rendered_url, hash_)
+
+    with io.StringIO() as f:
+        yaml.dump(data, f)
+        return f.getvalue()

src/rattler_build_conda_compat/recipe_sources.py

@@ -18,12 +18,14 @@
 
 
 class Source(TypedDict):
-    url: NotRequired[str]
+    url: NotRequired[str | list[str]]
+    sha256: NotRequired[str]
+    md5: NotRequired[str]
 
 
-def get_all_url_sources(recipe: Mapping[Any, Any]) -> Iterator[str]:
+def get_all_sources(recipe: Mapping[Any, Any]) -> Iterator[Source]:
     """
-    Get all url sources from the recipe. This can be from a list of sources,
+    Get all sources from the recipe. This can be from a list of sources,
     a single source, or conditional and its branches.
 
     Arguments
@@ -32,18 +34,16 @@ def get_all_url_sources(recipe: Mapping[Any, Any]) -> Iterator[str]:
 
     Returns
     -------
-    A list of sources.
+    A list of source objects.
     """
-
     sources = recipe.get("source", None)
     sources = typing.cast(ConditionalList[Source], sources)
 
     # Try getting all url top-level sources
     if sources is not None:
         source_list = visit_conditional_list(sources, None)
         for source in source_list:
-            if url := source.get("url"):
-                yield url
+            yield source
 
     outputs = recipe.get("outputs", None)
     if outputs is None:
@@ -57,5 +57,26 @@ def get_all_url_sources(recipe: Mapping[Any, Any]) -> Iterator[str]:
             continue
         source_list = visit_conditional_list(sources, None)
         for source in source_list:
-            if url := source.get("url"):
-                yield url
+            yield source
+
+
+def get_all_url_sources(recipe: Mapping[Any, Any]) -> Iterator[str]:
+    """
+    Get all url sources from the recipe. This can be from a list of sources,
+    a single source, or conditional and its branches.
+
+    Arguments
+    ---------
+    * `recipe` - The recipe to inspect. This should be a yaml object.
+
+    Returns
+    -------
+    A list of URLs.
+    """
+
+    def get_first_url(source: Mapping[str, Any]) -> str:
+        if isinstance(source["url"], list):
+            return source["url"][0]
+        return source["url"]
+
+    return (get_first_url(source) for source in get_all_sources(recipe) if "url" in source)

tests/conftest.py

@@ -7,6 +7,11 @@
 import pytest
 
 
+@pytest.fixture()
+def data_dir() -> Path:
+    return Path(__file__).parent / "data"
+
+
 @pytest.fixture()
 def python_recipe(tmpdir: Path) -> str:
     recipe_dir = tmpdir / "recipe"

tests/data/build_number/test_1/expected.yaml

@@ -0,0 +1,10 @@
+# set the build number to something
+context:
+  build: 0
+
+package:
+  name: recipe_1
+  version: "0.1.0"
+
+build:
+  number: ${{ build }}

tests/data/build_number/test_1/recipe.yaml

@@ -0,0 +1,10 @@
+# set the build number to something
+context:
+  build: 123
+
+package:
+  name: recipe_1
+  version: "0.1.0"
+
+build:
+  number: ${{ build }}

tests/data/build_number/test_2/expected.yaml

@@ -0,0 +1,11 @@
+# set the build number to something
+package:
+  name: recipe_1
+  version: "0.1.0"
+
+# set the build number to something directly in the recipe text
+build:
+  number: 0
+
+source:
+  - url: foo

tests/data/build_number/test_2/recipe.yaml

@@ -0,0 +1,11 @@
+# set the build number to something
+package:
+  name: recipe_1
+  version: "0.1.0"
+
+# set the build number to something directly in the recipe text
+build:
+  number: 321
+
+source:
+- url: foo

tests/data/version/test_1/expected.yaml

@@ -0,0 +1,12 @@
+context:
+  name: xtensor
+  version: "0.25.0"
+
+
+package:
+  name: ${{ name|lower }}
+  version: ${{ version }}
+
+source:
+  url: https://github.com/xtensor-stack/xtensor/archive/${{ version }}.tar.gz
+  sha256: 32d5d9fd23998c57e746c375a544edf544b74f0a18ad6bc3c38cbba968d5e6c7

tests/data/version/test_1/recipe.yaml

@@ -0,0 +1,12 @@
+context:
+  name: xtensor
+  version: "0.23.5"
+
+
+package:
+  name: ${{ name|lower }}
+  version: ${{ version }}
+
+source:
+  url: https://github.com/xtensor-stack/xtensor/archive/${{ version }}.tar.gz
+  sha256: 0811011e448628f0dfa6ebb5e3f76dc7bf6a15ee65ea9c5a277b12ea976d35bc

tests/data/version/test_2/expected.yaml

@@ -0,0 +1,15 @@
+context:
+  name: xtensor
+  version: "0.25.0"
+
+
+package:
+  name: ${{ name|lower }}
+  version: ${{ version }}
+
+source:
+  # please update the version here.
+  - if: target_platform == linux-64
+    then:
+      url: https://github.com/xtensor-stack/xtensor/archive/${{ version }}.tar.gz
+      sha256: 32d5d9fd23998c57e746c375a544edf544b74f0a18ad6bc3c38cbba968d5e6c7

tests/data/version/test_2/recipe.yaml

@@ -0,0 +1,15 @@
+context:
+  name: xtensor
+  version: "0.23.5"
+
+
+package:
+  name: ${{ name|lower }}
+  version: ${{ version }}
+
+source:
+  # please update the version here.
+  - if: target_platform == linux-64
+    then:
+      url: https://github.com/xtensor-stack/xtensor/archive/${{ version }}.tar.gz
+      sha256: 0811011e448628f0dfa6ebb5e3f76dc7bf6a15ee65ea9c5a277b12ea976d35bc