Merge pull request #32 from aiyengar2/rebase_project_monitoring

Rebase rancher-project-monitoring against the latest rancher-monitoring chart

.github/workflows/e2e-ci.yaml

@@ -0,0 +1,143 @@
+name: E2E Prometheus Federator
+
+on:
+  workflow_dispatch:
+    inputs:
+      enable_tmate:
+        description: 'Enable debugging via tmate'
+        required: false
+        default: "false"
+      debug:
+        description: "Enable debug logs"
+        required: false
+        default: "false"
+      k3s_version:
+        description: "Version of k3s to use for the underlying cluster, should exist in https://hub.docker.com/r/rancher/k3s/tags"
+        required: false
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'scripts/**'
+      - '*.md'
+      - '*.dapper'
+      - '.gitignore'
+      - 'CODEOWNERS'
+      - 'LICENSE'
+      - 'Makefile'
+
+env:
+  GOARCH: amd64
+  CGO_ENABLED: 0
+  SETUP_GO_VERSION: '^1.18'
+  YQ_VERSION: v4.25.1
+  E2E_CI: true
+  REPO: rancher
+  TAG: dev
+  APISERVER_PORT: 8001
+  DEFAULT_SLEEP_TIMEOUT_SECONDS: 10
+  KUBECTL_WAIT_TIMEOUT: 120s
+  DEBUG: ${{ github.event.inputs.debug || false }}
+
+permissions:
+  contents: write
+
+jobs:
+  e2e-prometheus-federator:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k3s_version:
+          # k3d version list k3s | sed 's/+/-/' | sort -h
+          - ${{ github.event.inputs.k3s_version || 'v1.20.15-k3s1' }}
+    steps:
+      -
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      -
+        name: Install mikefarah/yq
+        run: |
+          sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq;
+      -
+        name: Perform CI
+        run: |
+          REPO=${REPO} TAG=${TAG} ./scripts/build;
+          REPO=${REPO} TAG=${TAG} ./scripts/package;
+      -
+        name: Provision k3d Cluster
+        uses: AbsaOSS/k3d-action@v2
+        # k3d will automatically create a network named k3d-test-cluster-1 with the range 172.18.0.0/16
+        with:
+          cluster-name: "e2e-ci-prometheus-federator"
+          args: >-
+            --agents 1
+            --network "nw01"
+            --image docker.io/rancher/k3s:${{matrix.k3s_version}}
+      -
+        name: Import Images Into k3d
+        run: |
+          k3d image import ${REPO}/prometheus-federator:${TAG} -c e2e-ci-prometheus-federator;
+      -
+        name: Setup kubectl context
+        run: |
+          kubectl config use-context k3d-e2e-ci-prometheus-federator;
+      -
+        name: Set Up Tmate Debug Session
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 15
+        with:
+          limit-access-to-actor: true
+      -
+        name: Install Rancher Monitoring
+        run: ./.github/workflows/e2e/scripts/install-monitoring.sh;
+      - 
+        name: Check if Rancher Monitoring is up
+        run: ./.github/workflows/e2e/scripts/validate-monitoring.sh;
+      -
+        name: Install Prometheus Federator
+        run: ./.github/workflows/e2e/scripts/install-federator.sh;
+      - 
+        name: Check if Prometheus Federator is up
+        run: ./.github/workflows/e2e/scripts/validate-federator.sh;
+      -
+        name: Check if Project Registration Namespace is auto-created on namespace detection
+        run: ./.github/workflows/e2e/scripts/create-project-namespace.sh;
+      -
+        name: Create Project Monitoring Stack via ProjectHelmChart CR
+        run: ./.github/workflows/e2e/scripts/create-projecthelmchart.sh;
+      - 
+        name: Check if the Project Prometheus Stack is up
+        run: ./.github/workflows/e2e/scripts/validate-project-monitoring.sh;
+      - 
+        name: Wait for 8 minutes for enough scraping to be done to continue
+        run: |
+          for i in {1..48}; do sleep 10; echo "Waited $((i*10)) seconds for metrics to be populated"...; done;
+      -
+        name: Validate Project Prometheus Targets
+        run: ./.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh;
+      -
+        name: Validate Project Grafana Datasources
+        run: ./.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh;
+      -
+        name: Validate Project Grafana Dashboards
+        run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh;
+      -
+        name: Validate Project Grafana Dashboard Data
+        run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh;
+      -
+        name: Validate Project Prometheus Alerts
+        run: ./.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh;
+      -
+        name: Validate Project Alertmanager
+        run: ./.github/workflows/e2e/scripts/validate-project-alertmanager.sh;
+      - 
+        name: Delete Project Prometheus Stack
+        run: ./.github/workflows/e2e/scripts/delete-projecthelmchart.sh;
+      - 
+        name: Uninstall Prometheus Federator
+        run: ./.github/workflows/e2e/scripts/uninstall-federator.sh;
+      - 
+        name: Delete k3d cluster
+        if: always()
+        run: k3d cluster delete e2e-ci-prometheus-federator

.github/workflows/e2e/package/Dockerfile

@@ -0,0 +1,17 @@
+FROM registry.suse.com/bci/golang:1.19-18.33 AS helm
+RUN zypper -n install git
+RUN git -C / clone --branch release-v3.9.0 --depth=1 https://github.com/rancher/helm
+RUN make -C /helm
+
+FROM registry.suse.com/bci/golang:1.17
+
+ARG ARCH=amd64
+ENV KUBECTL_VERSION v1.21.8
+
+# Install dependencies
+RUN zypper -n install awk git docker curl wget nodejs sudo
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl && \
+    chmod +x kubectl && mv ./kubectl /usr/local/bin/kubectl
+COPY --from=helm ./helm/bin/helm /usr/local/bin/helm
+
+CMD ["echo", "e2e-prometheus-federator"]

.github/workflows/e2e/scripts/create-project-namespace.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+kubectl create namespace e2e-prometheus-federator || true
+kubectl label namespace e2e-prometheus-federator field.cattle.io/projectId=p-example --overwrite
+sleep "${DEFAULT_SLEEP_TIMEOUT_SECONDS}"
+if ! kubectl get namespace cattle-project-p-example; then
+    echo "ERROR: Expected cattle-project-p-example namespace to exist after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds, not found"
+    exit 1
+fi
+
+echo "PASS: Project Registration Namespace was created"

.github/workflows/e2e/scripts/create-projecthelmchart.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+if [[ "${E2E_CI}" == "true" ]]; then
+    kubectl apply -f ./examples/ci-example.yaml
+else
+    kubectl apply -f ./examples/example.yaml
+fi
+sleep ${DEFAULT_SLEEP_TIMEOUT_SECONDS};
+
+if ! kubectl get -n cattle-monitoring-system job/helm-install-cattle-project-p-example-monitoring; then
+    echo "ERROR: Helm Install Job for Project Monitoring Stack was never created after ${KUBECTL_WAIT_TIMEOUT} seconds"
+    echo "PROJECT HELM CHARTS:"
+    kubectl get projecthelmchart -n cattle-project-p-example -o yaml
+    echo "HELM CHARTS:"
+    kubectl get helmcharts -n cattle-monitoring-system -o yaml
+    echo "HELM RELEASES:"
+    kubectl get helmreleases -n cattle-monitoring-system -o yaml
+    echo "PROMETHEUS FEDERATOR:"
+    kubectl logs deployment/prometheus-federator -n cattle-monitoring-system
+    exit 1
+fi
+
+if ! kubectl wait --for=condition=complete --timeout="${KUBECTL_WAIT_TIMEOUT}" -n cattle-monitoring-system job/helm-install-cattle-project-p-example-monitoring; then
+    echo "ERROR: Helm Install Job for Project Monitoring Stack never completed after ${KUBECTL_WAIT_TIMEOUT} seconds"
+    kubectl logs job/helm-install-cattle-project-p-example-monitoring -n cattle-monitoring-system
+    exit 1
+fi
+kubectl logs job/helm-install-cattle-project-p-example-monitoring -n cattle-monitoring-system
+
+echo "PASS: Adding ProjectHelmChart successfully installed Project Prometheus Stack"

.github/workflows/e2e/scripts/delete-projecthelmchart.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+kubectl delete -f ./examples/ci-example.yaml
+if kubectl get -n cattle-monitoring-system job/helm-delete-cattle-project-p-example-monitoring --ignore-not-found; then
+    if ! kubectl wait --for=condition=complete --timeout="${KUBECTL_WAIT_TIMEOUT}" -n cattle-monitoring-system job/helm-delete-cattle-project-p-example-monitoring; then
+        echo "ERROR: Helm Uninstall Job for Project Monitoring Stack never completed after ${KUBECTL_WAIT_TIMEOUT}"
+        kubectl logs job/helm-delete-cattle-project-p-example-monitoring -n cattle-monitoring-system
+        exit 1
+    fi
+fi
+
+if [[ $(kubectl get -n cattle-project-p-example -l "release=cattle-project-p-example-monitoring" secrets -o jsonpath='{.items[].metadata.name}' --ignore-not-found) != "cattle-project-p-example-m-alertmanager-secret" ]]; then
+    echo "ERROR: Expected Project Alertmanager Secret to be left behind in the namespace"
+    exit 1
+fi
+
+if [[ -n $(kubectl get -n cattle-project-p-example -l "release=cattle-project-p-example-monitoring" pods -o jsonpath='{.items[].metadata.name}' --ignore-not-found) ]]; then
+    echo "ERROR: Expected all pods of the Helm Release to be cleaned up on deleting the ProjectHelmChart"
+    exit 1
+fi
+
+echo "PASS: Removing ProjectHelmChart successfully uninstalled Project Prometheus Stack"

.github/workflows/e2e/scripts/entry

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)/../../../..
+
+DEFAULT_SLEEP_TIMEOUT_SECONDS=${DEFAULT_SLEEP_TIMEOUT_SECONDS:-10}
+KUBECTL_WAIT_TIMEOUT=${KUBECTL_WAIT_TIMEOUT:-120s}
+
+if [[ ${DEBUG} == "true" ]]; then
+    echo "Enabling DEBUG mode..."
+    set -x
+fi
+
+if [[ "${E2E_CI}" == "true" ]]; then
+    KUBERNETES_DISTRIBUTION_TYPE=k3s
+fi
+
+if [[ -n ${RANCHER_URL} ]] && [[ -n ${RANCHER_CLUSTER} ]] && [[ -n ${RANCHER_TOKEN} ]]; then
+    API_SERVER_URL=${RANCHER_URL}/k8s/clusters/${RANCHER_CLUSTER}
+    API_SERVER_CURL_AUTH_HEADERS="-k -H 'Authorization: Bearer ${RANCHER_TOKEN}'"
+    RANCHER_HELM_ARGS="--set global.cattle.url=${RANCHER_URL} --set global.cattle.clusterId=${RANCHER_CLUSTER}"
+else
+    kubectl proxy --port=${APISERVER_PORT:-8001} 2>/dev/null &
+    API_SERVER_URL=http://localhost:${APISERVER_PORT:-8001}
+    sleep 5
+fi

.github/workflows/e2e/scripts/install-federator.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+latest_chart=$(find ./charts/prometheus-federator -type d -maxdepth 1 -mindepth 1 | tr - \~ | sort -rV | tr \~ - | head -n1)
+
+case "${KUBERNETES_DISTRIBUTION_TYPE}" in
+"k3s")
+    cluster_args=""
+    kubernetes_version=$(kubectl version --short | grep "Server Version" | cut -d ' ' -f3)
+    case "${kubernetes_version}" in
+    v1.23.*)
+        embedded_helm_controller_fixed_version="v1.23.14"
+        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
+            cluster_args="--set helmProjectOperator.helmController.enabled=false"
+        fi
+        ;;
+    v1.24.*)
+        embedded_helm_controller_fixed_version="v1.24.8"
+        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
+            cluster_args="--set helmProjectOperator.helmController.enabled=false"
+        fi
+        ;;
+    *)
+        embedded_helm_controller_fixed_version="v1.25.4"
+        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
+            cluster_args="--set helmProjectOperator.helmController.enabled=false"
+        fi
+        ;;
+    esac
+    ;;
+"rke")
+    cluster_args=""
+    ;;
+"rke2")
+    cluster_args=""
+    kubernetes_version=$(kubectl version --short | grep "Server Version" | cut -d ' ' -f3)
+    case "${kubernetes_version}" in
+    v1.23.*)
+        embedded_helm_controller_fixed_version="v1.23.14"
+        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
+            cluster_args="--set helmProjectOperator.helmController.enabled=false"
+        fi
+        ;;
+    v1.24.*)
+        embedded_helm_controller_fixed_version="v1.24.8"
+        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
+            cluster_args="--set helmProjectOperator.helmController.enabled=false"
+        fi
+        ;;
+    *)
+        embedded_helm_controller_fixed_version="v1.25.4"
+        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
+            cluster_args="--set helmProjectOperator.helmController.enabled=false"
+        fi
+        ;;
+    esac
+    ;;
+*)
+    echo "KUBERNETES_DISTRIBUTION_TYPE=${KUBERNETES_DISTRIBUTION_TYPE} is unknown"
+    exit 1
+esac
+
+helm upgrade --install --create-namespace -n cattle-monitoring-system prometheus-federator --set helmProjectOperator.image.repository=${REPO:-rancher}/prometheus-federator --set helmProjectOperator.image.tag=${TAG:-dev} ${cluster_args} ${RANCHER_HELM_ARGS} ${latest_chart}
+
+echo "PASS: Prometheus Federator has been installed"

.github/workflows/e2e/scripts/install-monitoring.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+helm repo add rancher-charts https://charts.rancher.io
+helm repo update
+helm upgrade --install --create-namespace -n cattle-monitoring-system rancher-monitoring-crd rancher-charts/rancher-monitoring-crd
+
+if [[ "${E2E_CI}" == "true" ]]; then
+    e2e_args="--set grafana.resources=null --set prometheus.prometheusSpec.resources=null --set alertmanager.alertmanagerSpec.resources=null"
+fi
+
+case "${KUBERNETES_DISTRIBUTION_TYPE}" in
+"k3s")
+    cluster_args="--set k3sServer.enabled=true"
+    ;;
+"rke")
+    cluster_args="--set rkeControllerManager.enabled=true --set rkeScheduler.enabled=true --set rkeProxy.enabled=true --set rkeEtcd.enabled=true"
+    ;;
+"rke2")
+    cluster_args="--set rke2ControllerManager.enabled=true --set rke2Scheduler.enabled=true --set rke2Proxy.enabled=true --set rke2Etcd.enabled=true"
+    ;;
+*)
+    echo "KUBERNETES_DISTRIBUTION_TYPE=${KUBERNETES_DISTRIBUTION_TYPE} is unknown"
+    exit 1
+esac
+
+helm upgrade --install --create-namespace -n cattle-monitoring-system rancher-monitoring ${cluster_args} ${e2e_args} ${RANCHER_HELM_ARGS} rancher-charts/rancher-monitoring
+
+echo "PASS: Rancher Monitoring has been installed"

.github/workflows/e2e/scripts/uninstall-federator.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+helm uninstall --wait -n cattle-monitoring-system prometheus-federator
+
+echo "PASS: Prometheus Federator has been uninstalled"

.github/workflows/e2e/scripts/validate-federator.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+if ! kubectl -n cattle-monitoring-system rollout status deployment prometheus-federator --timeout="${KUBECTL_WAIT_TIMEOUT}"; then 
+    echo "ERROR: Prometheus Federator did not roll out"
+    kubectl get pods -n cattle-monitoring-system -o yaml
+    exit 1
+fi
+
+echo "PASS: Prometheus Federator is up and running"