Cross-controller integration testing for opensearch

.github/workflows/integration_test.yaml

@@ -27,3 +27,44 @@ jobs:
           channel: 1.25-strict/stable
       - name: Run integration tests
         run: tox -e ${{ matrix.tox-environments }}
+  integration-test-cross-controller:
+    name: Integration tests (microk8s & lxd)
+    strategy:
+      matrix:
+        tox-environments:
+          - integration-opensearch
+    runs-on: ["self-hosted", "linux", "X64", "jammy", "large"]
+    timeout-minutes: 120
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Get prefsrc
+        run: |
+          echo "IPADDR=$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc')" >> $GITHUB_ENV
+      - name: Setup K8s environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          juju-channel: 3.1/stable
+          provider: microk8s
+          microk8s-addons: "hostpath-storage dns metallb:${{ env.IPADDR }}-${{ env.IPADDR }}"
+          channel: 1.26-strict/stable
+      - name: Save K8s controller name
+        id: k8s-controller
+        run: echo "name=$CONTROLLER_NAME" >> $GITHUB_OUTPUT
+      - name: Setup LXD environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          juju-channel: 3.1/stable
+          provider: lxd
+      - name: Save LXD controller name
+        id: lxd-controller
+        run: echo "name=$CONTROLLER_NAME" >> $GITHUB_OUTPUT
+      - name: Run integration tests
+        run: tox -e ${{ matrix.tox-environments }}
+        env:
+          K8S_CONTROLLER: ${{ steps.k8s-controller.outputs.name }}
+          LXD_CONTROLLER: ${{ steps.lxd-controller.outputs.name }}
+      - name: Dump debug log
+        if: failure()
+        run: for ctl in $(juju controllers --format json | jq -r '.controllers | keys[]'); do for mdl in $(juju models --format json | jq -r '.models[].name' | grep -v "admin/controller"); do juju debug-log -m $ctl:$mdl --replay --ms --no-tail; done; done || true
+        shell: bash

tests/integration/helpers.py

@@ -7,6 +7,8 @@
 import json
 import logging
 from pathlib import Path
+from juju.controller import Controller
+from juju.model import Model
 
 import requests
 import yaml
@@ -130,3 +132,29 @@ async def get_memberships(ops_test: OpsTest, url):
     user_id = data["vXGroupUsers"][0].get("userId")
     membership = (group, user_id)
     return membership
+
+
+async def get_or_add_model(
+    ops_test: OpsTest, controller: Controller, model_name: str
+) -> Model:
+    """Add model if it does not exist and get model.
+
+    Args:
+        ops_test: PyTest object.
+        controller: Juju controller.
+        model_name: Juju model name.
+
+    Returns:
+        Juju model by name.
+    """
+    if model_name not in await controller.get_models():
+        await controller.add_model(model_name)
+        controller_name = controller.controller_name
+        await ops_test.track_model(
+            f"{controller_name}-{model_name}",
+            cloud_name=controller_name,
+            model_name=model_name,
+            keep=False,
+        )
+        logger.info(f"model {model_name} created")
+    return await controller.get_model(model_name)

tests/integration/test_opensearch.py

@@ -0,0 +1,233 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""Opensearch cross-controller integration test."""
+
+import asyncio
+import logging
+import os
+
+import pytest
+import pytest_asyncio
+import requests
+from helpers import (
+    APP_NAME,
+    LXD_MODEL_CONFIG,
+    METADATA,
+    POSTGRES_NAME,
+    get_or_add_model,
+)
+from juju.controller import Controller
+from juju.model import Model
+from pytest_operator.plugin import OpsTest
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.skip_if_deployed
+@pytest_asyncio.fixture(name="deploy-opensearch", scope="module")
+async def setup_models(ops_test: OpsTest):
+    """Setup controllers and models.
+
+    Args:
+        ops_test: PyTest object.
+    """
+    lxd_controller_name = os.environ["LXD_CONTROLLER"]
+    k8s_controller_name = os.environ["K8S_CONTROLLER"]
+
+    k8s_model_name = lxd_model_name = ops_test.model_name
+
+    logger.info(
+        f"connecting to k8s controller and creating model {k8s_model_name}"
+    )
+    k8s_controller = Controller()
+    await k8s_controller.connect(k8s_controller_name)
+    k8s_model = await get_or_add_model(
+        ops_test, k8s_controller, k8s_model_name
+    )
+    await k8s_model.set_config(
+        {"logging-config": "<root>=WARNING; unit=DEBUG"}
+    )
+
+    logger.info(
+        f"connecting to lxd controller and creating model {lxd_model_name}"
+    )
+    lxd_controller = Controller()
+    await lxd_controller.connect(lxd_controller_name)
+    lxd_model = await get_or_add_model(
+        ops_test, lxd_controller, lxd_model_name
+    )
+    await lxd_model.set_config(LXD_MODEL_CONFIG)
+    await deploy_opensearch(lxd_model)
+    await deploy_ranger(ops_test, k8s_model)
+
+    await integrate_ranger_opensearch(
+        ops_test, k8s_model, lxd_model, lxd_controller_name
+    )
+
+
+async def deploy_opensearch(lxd_model: Model):
+    """Deploy OpenSearch and related components.
+
+    Args:
+        lxd_model: The LXD model.
+    """
+    logger.info("deploying opensearch")
+    await asyncio.gather(
+        lxd_model.deploy(
+            "ch:opensearch",
+            num_units=2,
+            channel="2/edge",
+        ),
+        lxd_model.deploy(
+            "self-signed-certificates", num_units=1, channel="edge"
+        ),
+    )
+    await lxd_model.wait_for_idle(
+        apps=["opensearch"],
+        status="blocked",
+        raise_on_blocked=False,
+        timeout=3000,
+    )
+    await lxd_model.add_relation("self-signed-certificates", "opensearch")
+    await lxd_model.wait_for_idle(
+        apps=["opensearch", "self-signed-certificates"],
+        status="active",
+        raise_on_blocked=False,
+        timeout=3000,
+    )
+    await lxd_model.create_offer("opensearch:opensearch-client")
+
+
+async def deploy_ranger(
+    ops_test: OpsTest,
+    k8s_model: Model,
+):
+    """Deploy Ranger and integrate with OpenSearch.
+
+    Args:
+        ops_test: PyTest object.
+        k8s_model: The K8s model for Ranger deployment.
+    """
+    await k8s_model.deploy(POSTGRES_NAME, channel="14", trust=True)
+    charm = await ops_test.build_charm(".")
+    resources = {
+        "ranger-image": METADATA["resources"]["ranger-image"][
+            "upstream-source"
+        ]
+    }
+    await k8s_model.deploy(
+        charm, resources=resources, application_name=APP_NAME
+    )
+    await k8s_model.wait_for_idle(
+        apps=[POSTGRES_NAME],
+        status="active",
+        raise_on_blocked=False,
+        timeout=1500,
+    )
+    await k8s_model.wait_for_idle(
+        apps=[APP_NAME],
+        status="blocked",
+        raise_on_blocked=False,
+        timeout=1500,
+    )
+
+    logger.info("Integrating Ranger and Postgresql")
+    await k8s_model.integrate(APP_NAME, POSTGRES_NAME)
+    await k8s_model.wait_for_idle(
+        apps=[POSTGRES_NAME, APP_NAME],
+        status="active",
+        raise_on_blocked=False,
+        timeout=1500,
+    )
+
+
+async def integrate_ranger_opensearch(
+    ops_test: OpsTest,
+    k8s_model: Model,
+    lxd_model: Model,
+    lxd_controller_name: str,
+):
+    """Integrate Ranegr and OpenSearch.
+
+    Args:
+        ops_test: PyTest object.
+        k8s_model: The K8s model for Ranger deployment.
+        lxd_model: The LXD model for OpenSearch deployment.
+        lxd_controller_name: The name of the LXD controller.
+    """
+    await k8s_model.consume(
+        f"admin/{lxd_model.name}.opensearch",
+        controller_name=lxd_controller_name,
+    )
+
+    logger.info("Integrating Ranger and OpenSearch")
+    await k8s_model.integrate(APP_NAME, "opensearch")
+    async with ops_test.fast_forward():
+        await k8s_model.wait_for_idle(
+            apps=[APP_NAME],
+            status="active",
+            raise_on_blocked=False,
+            timeout=1500,
+        )
+
+
+@pytest.mark.abort_on_fail
+@pytest.mark.usefixtures("deploy-opensearch")
+class TestOpenSearch:
+    """Integration tests for auditing Ranger charm."""
+
+    async def test_ranger_audits(self, ops_test: OpsTest):
+        """Perform GET request on the Ranger audits endpoint."""
+        logger.info("opensearch and ranger successfully related.")
+
+        k8s_controller_name = os.environ["K8S_CONTROLLER"]
+        k8s_controller = Controller()
+        await k8s_controller.connect(k8s_controller_name)
+        k8s_model_name = ops_test.model_name
+        k8s_model = await get_or_add_model(
+            ops_test, k8s_controller, k8s_model_name
+        )
+
+        status = await k8s_model.get_status()  # noqa: F821
+        logger.info(status)
+        address = status["applications"][APP_NAME]["units"][f"{APP_NAME}/0"][
+            "address"
+        ]
+        url = f"http://{address}:6080"
+
+        audit_url = f"{url}/service/assets/accessAudit"
+        logger.info("curling app address: %s", audit_url)
+
+        try:
+            response = requests.get(
+                audit_url, timeout=300, verify=False
+            )  # nosec
+            assert response.status_code == 200
+        except Exception as e:
+            logger.debug(e)
+
+        logger.info("Removing Ranger and Saas")
+        await k8s_model.applications[APP_NAME].remove_relation(f"{APP_NAME}:opensearch", "opensearch:opensearch-client")
+
+
+        await asyncio.gather(
+            k8s_model.remove_application(APP_NAME, block_until_done=True),
+            k8s_model.remove_application(POSTGRES_NAME, block_until_done=True),
+        )
+        await asyncio.gather(k8s_model.remove_saas("opensearch"))
+
+        # logger.info("Removing Opensearch")
+        # lxd_controller_name = os.environ["LXD_CONTROLLER"]
+        # lxd_controller = Controller()
+        # await lxd_controller.connect(lxd_controller_name)
+        # lxd_model = await get_or_add_model(
+        #     ops_test, lxd_controller, ops_test.model_name
+        # )
+
+        # await asyncio.gather(
+        #     lxd_model.remove_application("opensearch"),
+        #     lxd_model.remove_application("self-signed-certificates"),
+        # )
+        # # Give it some time to settle, since we cannot block until complete.
+        # await asyncio.sleep(180)

tox.ini

@@ -95,6 +95,26 @@ deps =
 commands =
     pytest {[vars]tst_path}integration/test_usersync.py -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs}
 
+[testenv:integration-opensearch]
+description = Run integration tests
+passenv =
+    K8S_CONTROLLER
+    LXD_CONTROLLER
+deps =
+    ipdb==0.13.9
+    juju==3.1.0.1
+    pytest==7.4.0
+    pytest-operator==0.29.0
+    pytest-asyncio==0.21
+    -r{toxinidir}/requirements.txt
+allowlist_externals =
+    sudo
+    sysctl
+commands_pre =
+    sudo sysctl -w vm.max_map_count=262144 vm.swappiness=0 net.ipv4.tcp_retries2=5
+commands =
+    pytest {[vars]tst_path}integration/test_opensearch.py -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs}
+
 [testenv:unit]
 description = Run tests
 deps =