✨ Introduce E2E setup and tests in GKE

.github/workflows/e2e-latest-rancher.yaml

@@ -0,0 +1,14 @@
+name: e2e-latest-rancher
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  schedule:
+    - cron: 0 22 * * *
+permissions: read-all
+jobs:
+  e2e-tests:
+    uses: ./.github/workflows/e2e.yaml
+    secrets:
+      GKE_CREDENTIALS: ${{ secrets.GKE_CREDENTIALS }}

.github/workflows/e2e.yaml

@@ -0,0 +1,79 @@
+name: E2E tests
+on:
+  workflow_call:
+    secrets:
+      GKE_CREDENTIALS:
+        description: "GKE credentials"
+        required: true
+jobs:
+  e2e-tests:
+    env:
+      REPO: ttl.sh/gke-operator-ci
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Export tag
+        id: export_tag
+        run: |
+          TAG=`git describe --abbrev=0 --tags 2>/dev/null || echo "v0.0.0"`
+          COMMITDATE=`date -d @$(git log -n1 --format="%at") "+%FT%TZ"`
+          echo "operator_tag=$TAG" >> $GITHUB_OUTPUT
+          echo "commit_date=$COMMITDATE" >> $GITHUB_OUTPUT
+      - name: Docker meta
+        id: meta
+        uses: docker/[email protected]
+        with:
+          images: |
+            ${{ env.REPO }}
+          tags: |
+            type=sha,format=short,prefix=${{ steps.export_tag.outputs.operator_tag }}-
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/[email protected]
+      - name: Build and push image
+        uses: docker/[email protected]
+        with:
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          target: gke-operator
+          file: test/e2e/Dockerfile.e2e
+          build-args: |
+            TAG=${{ steps.export_tag.outputs.operator_tag }}
+            COMMITDATE=${{ steps.export_tag.outputs.commit_date }}
+            COMMIT=${{ github.sha }}
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.20.x
+      - uses: actions/[email protected]
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+          key: ${{ runner.os }}-test-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-test-go-${{ hashFiles('**/go.sum') }}
+      - uses: engineerd/[email protected]
+        with:
+          version: "v0.16.0"
+          skipClusterCreation: "true"
+      - name: Create kind cluster
+        run: make setup-kind
+      - name: E2E tests
+        env:
+          GKE_CREDENTIALS: "${{ secrets.GKE_CREDENTIALS }}"
+        run: make e2e-tests
+      - name: Archive artifacts
+        if: always()
+        uses: actions/[email protected]
+        with:
+          name: ci-artifacts
+          path: _artifacts
+          if-no-files-found: ignore

.github/workflows/update-rancher-charts.yaml

@@ -1,4 +1,4 @@
-name: Update Operator in rancher/charts
+name: Update GKE operator in rancher/charts
 on:
   workflow_dispatch:
     inputs:

.github/workflows/update-rancher-dep.yaml

@@ -1,4 +1,4 @@
-name: Update operator in rancher/rancher
+name: Update GKE operator in rancher/rancher
 on:
   workflow_dispatch:
     inputs:

.gitignore

@@ -4,3 +4,4 @@ bin/
 dist/
 vendor/
 .dapper
+_artifacts/

Makefile

@@ -1,4 +1,14 @@
 TARGETS := $(shell ls scripts)
+GIT_COMMIT?=$(shell git rev-parse HEAD)
+GIT_COMMIT_SHORT?=$(shell git rev-parse --short HEAD)
+GIT_TAG?=$(shell git describe --abbrev=0 --tags 2>/dev/null || echo "v0.0.0" )
+TAG?=${GIT_TAG}-${GIT_COMMIT_SHORT}
+OPERATOR_CHART?=$(shell find $(ROOT_DIR) -type f -name "rancher-gke-operator-[0-9]*.tgz" -print)
+CRD_CHART?=$(shell find $(ROOT_DIR) -type f -name "rancher-gke-operator-crd*.tgz" -print)
+CHART_VERSION?=900 # Only used in e2e to avoid downgrades from rancher
+REPO?=ghcr.io/rancher/gke-operator
+CLUSTER_NAME?="gke-operator-e2e"
+E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/config.yaml
 
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 BIN_DIR := $(abspath $(ROOT_DIR)/bin)
@@ -12,30 +22,31 @@ MOCKGEN_PKG := github.com/golang/mock/mockgen
 GINKGO_VER := v2.13.2
 GINKGO_BIN := ginkgo
 GINKGO := $(BIN_DIR)/$(GINKGO_BIN)-$(GINKGO_VER)
+GINKGO_PKG := github.com/onsi/ginkgo/v2/ginkgo
 
-GO_APIDIFF_VER := v0.6.0
+GO_APIDIFF_VER := v0.7.0
 GO_APIDIFF_BIN := go-apidiff
 GO_APIDIFF := $(BIN_DIR)/$(GO_APIDIFF_BIN)-$(GO_APIDIFF_VER)
 GO_APIDIFF_PKG := github.com/joelanford/go-apidiff
 
+default: operator
+
+.dapper:
+	@echo Downloading dapper
+	@curl -sL https://releases.rancher.com/dapper/latest/dapper-`uname -s`-`uname -m` > .dapper.tmp
+	@@chmod +x .dapper.tmp
+	@./.dapper.tmp -v
+	@mv .dapper.tmp .dapper
 
 $(MOCKGEN):
 	GOBIN=$(BIN_DIR) $(GO_INSTALL) $(MOCKGEN_PKG) $(MOCKGEN_BIN) $(MOCKGEN_VER)
 
 $(GINKGO):
-	GOBIN=$(BIN_DIR) $(GO_INSTALL) github.com/onsi/ginkgo/v2/ginkgo $(GINKGO_BIN) $(GINKGO_VER)
+	GOBIN=$(BIN_DIR) $(GO_INSTALL) $(GINKGO_PKG) $(GINKGO_BIN) $(GINKGO_VER)
 
 $(GO_APIDIFF):
 	GOBIN=$(BIN_DIR) $(GO_INSTALL) $(GO_APIDIFF_PKG) $(GO_APIDIFF_BIN) $(GO_APIDIFF_VER)
 
-
-.dapper:
-	@echo Downloading dapper
-	@curl -sL https://releases.rancher.com/dapper/latest/dapper-`uname -s`-`uname -m` > .dapper.tmp
-	@@chmod +x .dapper.tmp
-	@./.dapper.tmp -v
-	@mv .dapper.tmp .dapper
-
 .PHONY: $(TARGETS)
 $(TARGETS): .dapper
 	./.dapper $@
@@ -73,6 +84,60 @@ verify-generate: generate
 		echo "generated files are out of date, run make generate"; exit 1; \
 	fi
 
+.PHONY: operator-chart
+operator-chart:
+	mkdir -p $(BIN_DIR)
+	cp -rf $(ROOT_DIR)/charts/gke-operator $(BIN_DIR)/chart
+	sed -i -e 's/tag:.*/tag: '${TAG}'/' $(BIN_DIR)/chart/values.yaml
+	sed -i -e 's|repository:.*|repository: '${REPO}'|' $(BIN_DIR)/chart/values.yaml
+	helm package --version ${CHART_VERSION} --app-version ${GIT_TAG} -d $(BIN_DIR)/ $(BIN_DIR)/chart
+	rm -Rf $(BIN_DIR)/chart
+
+.PHONY: crd-chart
+crd-chart:
+	mkdir -p $(BIN_DIR)
+	helm package --version ${CHART_VERSION} --app-version ${GIT_TAG} -d $(BIN_DIR)/ $(ROOT_DIR)/charts/gke-operator-crd
+	rm -Rf $(BIN_DIR)/chart
+
+.PHONY: charts
+charts:
+	$(MAKE) operator-chart
+	$(MAKE) crd-chart
+
+.PHONY: setup-kind
+setup-kind:
+	CLUSTER_NAME=$(CLUSTER_NAME) $(ROOT_DIR)/scripts/setup-kind-cluster.sh
+
+.PHONY: e2e-tests
+e2e-tests: $(GINKGO) charts
+	export EXTERNAL_IP=`kubectl get nodes -o jsonpath='{.items[].status.addresses[?(@.type == "InternalIP")].address}'` && \
+	export BRIDGE_IP="172.18.0.1" && \
+	export CONFIG_PATH=$(E2E_CONF_FILE) && \
+	export OPERATOR_CHART=$(OPERATOR_CHART) && \
+	export CRD_CHART=$(CRD_CHART) && \
+	cd $(ROOT_DIR)/test && $(GINKGO) $(ONLY_DEPLOY) -r -v ./e2e
+
+.PHONY: kind-e2e-tests
+kind-e2e-tests: docker-build-e2e setup-kind
+	kind load docker-image --name $(CLUSTER_NAME) ${REPO}:${TAG}
+	$(MAKE) e2e-tests
+
+kind-deploy-operator:
+	ONLY_DEPLOY="--label-filter=\"do-nothing\"" $(MAKE) kind-e2e-tests
+
+.PHONY: docker-build
+docker-build-e2e:
+	DOCKER_BUILDKIT=1 docker build \
+		-f test/e2e/Dockerfile.e2e \
+		--build-arg "TAG=${GIT_TAG}" \
+		--build-arg "COMMIT=${GIT_COMMIT}" \
+		--build-arg "COMMITDATE=${COMMITDATE}" \
+		-t ${REPO}:${TAG} .
+
+.PHOHY: delete-local-kind-cluster
+delete-local-kind-cluster: ## Delete the local kind cluster
+	kind delete cluster --name=$(CLUSTER_NAME)
+
 .PHONY: clean
 clean:
 	rm -rf build bin dist

README.md

@@ -1,70 +1,87 @@
 # gke-operator
 
-GKE operator is a Kubernetes CRD controller that controls cluster provisioning in Google Kubernetes Engine using an GKEClusterConfig defined by a Custom Resource Definition.
+The GKE operator is a controller for Kubernetes Custom Resource Definitions (CRDs) that manages cluster provisioning in Google Kubernetes Engine. It uses a GKEClusterConfig defined by a CRD.
 
 ## Build
 
+Operator binary can be built using the following command:
+
 ```sh
-go build -o gke-operator main.go
+    make operator
 ```
 
-## Test
+## Deploy operator from source
 
-With `KUBECONFIG` set in your shell, run the binary
+You can use the following command to deploy a Kind cluster with Rancher manager and operator:
 
 ```sh
-./gke-operator
+    make kind-deploy-operator
 ```
 
-Apply the CRD
+After this, you can also scale down operator deployment and run it from a local binary.
+
+## Tests
+
+To run unit tests use the following command:
 
 ```sh
-kubectl apply -f crds/gkeclusterconfig.yaml
+    make test
 ```
 
-Create a file named `googlecredentialConfig-authEncodedJson` with the contents
-of your JSON service account credential. Then create a cloud credential secret:
+## E2E
+
+We run e2e tests after every merged PR and periodically every 24 hours. They are triggered by a [Github action](https://github.com/rancher/gke-operator/blob/main/.github/workflows/e2e-latest-rancher.yaml)
+
+For running e2e tests:
+
+1. Set at least a `projectID` to valid projectID in [GKEClusterConfig](./test/e2e/templates/basic-cluster.yaml) definition, which is used to provision a GKE cluster 
+2. Set `GKE_CREDENTIALS` environment variable by providing a full path to valid gke credentials JSON:
 
 ```sh
-kubectl --namespace cattle-global-data create secret generic --from-file=googlecredentialConfig-authEncodedJson cc-abcde
+    export GKE_CREDENTIALS=$( cat /path/to/gke-credentials.json )
 ```
 
-Edit at a minimum the `projectID` and create a cluster
+3. and finally run:
 
 ```sh
-kubectl apply -f examples/cluster-basic.yaml
+    make kind-e2e-tests
 ```
 
-## Develop
+This will setup a kind cluster locally, and the e2e tests will be run against where it will:
 
-The easiest way to debug and develop the GKE operator is to replace the default operator on a running Rancher instance with your local one.
+    * deploy rancher and cert-manager
+    * deploy gke operator and operator CRD charts
+    * create gke credentials secret
+    * create a cluster in GKE
+    * wait for cluster to be ready
+    * clean up cluster
 
-* Run a local Rancher server
-* Provision a GKE cluster
-* Scale the gke-operator deployment to replicas=0 in the Rancher UI
-* Open the gke-operator repo in Goland, set `KUBECONFIG=<kubeconfig_path>` in Run Configuration Environment
-* Run the gke-operator in Debug Mode
-* Set breakpoints
+
+Once e2e tests are completed, the local kind cluster can also be deleted by running:
+
+```bash
+    make delete-local-kind-cluster
+```
 
 ## Release
 
-#### When should I release?
+### When should I release?
 
-A KEv2 operator should be released if
+A KEv2 operator should be released if:
 
 * There have been several commits since the last release,
 * You need to pull in an update/bug fix/backend code to unblock UI for a feature enhancement in Rancher
 * The operator needs to be unRC for a Rancher release
 
-#### How do I release?
+### How do I release?
 
 Tag the latest commit on the `master` branch. For example, if latest tag is:
 * `v1.1.3-rc1` you should tag `v1.1.3-rc2`.
 * `v1.1.3` you should tag `v1.1.4-rc1`.
 
 ```bash
 # Get the latest upstream changes
-# Note: `upstream` must be the remote pointing to `[email protected]:rancher/eks-operator.git`.
+# Note: `upstream` must be the remote pointing to `[email protected]:rancher/gke-operator.git`.
 git pull upstream master --tags
 
 # Export the tag of the release to be cut, e.g.:
@@ -74,15 +91,17 @@ export RELEASE_TAG=v1.1.3-rc2
 git tag -s -a ${RELEASE_TAG} -m ${RELEASE_TAG}
 
 # Push tags
-# Note: `upstream` must be the remote pointing to `[email protected]:rancher/eks-operator.git`.
+# Note: `upstream` must be the remote pointing to `[email protected]:rancher/gke-operator.git`.
 git push upstream ${RELEASE_TAG}
 ```
 
-Submit a [rancher/charts PR](https://github.com/rancher/charts/pull/2242) to update the operator and operator-crd chart versions.
-Submit a [rancher/rancher PR](https://github.com/rancher/rancher/pull/39745) to update the bundled chart.
+After pushing the release tag, you need to run 2 Github Actions. You can find them in the Actions tab of the repo:
+
+* [Update GKE operator in rancher/rancher](https://github.com/rancher/gke-operator/actions/workflows/update-rancher-dep.yaml) - This action will update the GKE operator in rancher/rancher repo. It will bump go dependencies.
+* [Update GKE operator in rancher/charts](https://github.com/rancher/gke-operator/actions/workflows/update-rancher-charts.yaml) - This action will update the GKE operator in rancher/charts repo. It will bump the chart version.
 
-#### How do I unRC?
+### How do I unRC?
 
-UnRC is the process of removing the rc from a KEv2 operator tag and means the released version is stable and ready for use. Release the KEv2 operator but instead of bumping the rc, remove the rc. For example, if the latest release of GKE operator is:
+unRC is the process of removing the rc from a KEv2 operator tag and means the released version is stable and ready for use. Release the KEv2 operator but instead of bumping the rc, remove the rc. For example, if the latest release of GKE operator is:
 * `v1.1.3-rc1`, release the next version without the rc which would be `v1.1.3`.
 * `v1.1.3`, that has no rc so release that version or `v1.1.4` if updates are available.