template-renderer: allow using variables in dynamic variables (#4427)

* template-renderer: create parent dirs for local file persistence

* template-renderer: allow using variables in dynamic variable queries

* template-renderer: add default dynamic variable query result

* template-renderer: unpack_dynamic_variables tests for templated dynamic variables

reconcile/templating/renderer.py

@@ -3,6 +3,7 @@
 import tempfile
 from abc import ABC, abstractmethod
 from collections.abc import Callable
+from pathlib import Path
 from typing import Any, Optional, Self
 
 from deepdiff import DeepHash
@@ -30,6 +31,7 @@
 from reconcile.utils import gql
 from reconcile.utils.git import clone
 from reconcile.utils.gql import init_from_config
+from reconcile.utils.jinja2.utils import process_jinja2_template
 from reconcile.utils.ruamel import create_ruamel_instance
 from reconcile.utils.runtime.integration import (
     PydanticRunParams,
@@ -85,12 +87,9 @@ def __init__(self, app_interface_data_path: str) -> None:
 
     def write(self, outputs: list[TemplateOutput]) -> None:
         for output in outputs:
-            with open(
-                f"{join_path(self.app_interface_data_path, output.path)}",
-                "w",
-                encoding="utf-8",
-            ) as f:
-                f.write(output.content)
+            filepath = Path(join_path(self.app_interface_data_path, output.path))
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+            filepath.write_text(output.content, encoding="utf-8")
 
     def read(self, path: str) -> Optional[str]:
         return self._read_local_file(join_path(self.app_interface_data_path, path))
@@ -166,12 +165,14 @@ def unpack_static_variables(
 def unpack_dynamic_variables(
     collection_variables: TemplateCollectionVariablesV1, gql: gql.GqlApi
 ) -> dict[str, dict[str, Any]]:
-    if not collection_variables.dynamic:
-        return {}
-
-    return {
-        dv.name: gql.query(dv.query) or {} for dv in collection_variables.dynamic or []
-    }
+    static = collection_variables.static or {}
+    dynamic: dict[str, dict[str, Any]] = {}
+    for dv in collection_variables.dynamic or []:
+        query = process_jinja2_template(
+            body=dv.query, vars={"static": static, "dynamic": dynamic}
+        )
+        dynamic[dv.name] = gql.query(query) or {}
+    return dynamic
 
 
 def calc_template_hash(c: TemplateCollectionV1, variables: dict[str, Any]) -> str:

reconcile/test/templating/test_renderer.py

@@ -10,6 +10,7 @@
 
 from reconcile.gql_definitions.templating.template_collection import (
     TemplateCollectionV1,
+    TemplateCollectionVariablesQueriesV1,
     TemplateCollectionVariablesV1,
     TemplateV1,
 )
@@ -27,6 +28,7 @@
     unpack_dynamic_variables,
     unpack_static_variables,
 )
+from reconcile.utils.jinja2.utils import Jinja2TemplateError
 from reconcile.utils.ruamel import create_ruamel_instance
 from reconcile.utils.secret_reader import SecretReader
 from reconcile.utils.vcs import VCS
@@ -167,6 +169,32 @@ def test_unpack_dynamic_variables_multiple_result(
     }
 
 
+def test_unpack_dynamic_variables_templated_query(
+    mocker: MockerFixture, collection_variables: TemplateCollectionVariablesV1
+) -> None:
+    gql = mocker.patch("reconcile.templating.renderer.gql.GqlApi", autospec=True)
+    collection_variables.dynamic = [
+        TemplateCollectionVariablesQueriesV1(
+            name="baz", query="templated query {{ static.foo }}"
+        )
+    ]
+    unpack_dynamic_variables(collection_variables, gql)
+    gql.query.assert_called_once_with("templated query bar")
+
+
+def test_unpack_dynamic_variables_templated_query_jinja_error(
+    mocker: MockerFixture, collection_variables: TemplateCollectionVariablesV1
+) -> None:
+    gql = mocker.patch("reconcile.templating.renderer.gql.GqlApi", autospec=True)
+    collection_variables.dynamic = [
+        TemplateCollectionVariablesQueriesV1(
+            name="foo", query="templated query {{ static.does_not_exist }}"
+        )
+    ]
+    with pytest.raises(Jinja2TemplateError):
+        unpack_dynamic_variables(collection_variables, gql)
+
+
 def test_join_path() -> None:
     assert join_path("foo", "bar") == "foo/bar"
     assert join_path("foo", "/bar") == "foo/bar"