Remove used providers in e2e

EKS cluster suffers from limited number of private IPs, which are exausted by test suites running in parallel. It affects GitRepo processing, where a pod created to update content of the repo in order to apply manifests to the cluster. Since the pod is not able to be scheduled, e2e tests are failing. Signed-off-by: Danil-Grigorev <[email protected]>
rancher · Jan 22, 2025 · 74debaa · 74debaa
1 parent 3d5dd5d
commit 74debaa
Show file tree

Hide file tree

Showing 10 changed files with 184 additions and 78 deletions.
diff --git a/Makefile b/Makefile
@@ -626,7 +626,7 @@ SKIP_RESOURCE_CLEANUP=$(SKIP_RESOURCE_CLEANUP) \
 USE_EXISTING_CLUSTER=$(USE_EXISTING_CLUSTER) \
 TURTLES_PATH=$(ROOT_DIR)/$(CHART_PACKAGE_DIR)/rancher-turtles-$(shell echo $(TAG) | cut -c 2-).tgz
 
-E2E_RUN_COMMAND=$(E2ECONFIG_VARS) $(GINKGO) -v --trace -p -procs=5 -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
+E2E_RUN_COMMAND=$(E2ECONFIG_VARS) $(GINKGO) -v --trace -p -procs=1 -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
 		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \
 		$(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(GINKGO_TESTS)

diff --git a/test/e2e/const.go b/test/e2e/const.go
@@ -33,8 +33,14 @@ var (
 	//go:embed data/capi-operator/capv-provider.yaml
 	CapvProvider []byte
 
-	//go:embed data/capi-operator/full-providers.yaml
-	FullProviders []byte
+	//go:embed data/capi-operator/aws-provider.yaml
+	AWSProvider []byte
+
+	//go:embed data/capi-operator/gcp-provider.yaml
+	GCPProvider []byte
+
+	//go:embed data/capi-operator/azure-provider.yaml
+	AzureProvider []byte
 
 	//go:embed data/capi-operator/capa-variables.yaml
 	AWSProviderSecret []byte

diff --git a/test/e2e/data/capi-operator/aws-provider.yaml b/test/e2e/data/capi-operator/aws-provider.yaml
@@ -0,0 +1,14 @@
+---
+apiVersion: turtles-capi.cattle.io/v1alpha1
+kind: CAPIProvider
+metadata:
+  name: aws
+  namespace: capa-system
+spec:
+  type: infrastructure
+  variables:
+    EXP_MACHINE_POOL: "true"
+    EXP_EXTERNAL_RESOURCE_GC: "true"
+    CAPA_LOGLEVEL: "4"
+  manager:
+    syncPeriod: "5m"
diff --git a/test/e2e/data/capi-operator/azure-provider.yaml b/test/e2e/data/capi-operator/azure-provider.yaml
@@ -0,0 +1,8 @@
+---
+apiVersion: turtles-capi.cattle.io/v1alpha1
+kind: CAPIProvider
+metadata:
+  name: azure
+  namespace: capz-system
+spec:
+  type: infrastructure
diff --git a/test/e2e/data/capi-operator/full-providers.yaml b/test/e2e/data/capi-operator/full-providers.yaml
diff --git a/test/e2e/data/capi-operator/gcp-provider.yaml b/test/e2e/data/capi-operator/gcp-provider.yaml
@@ -0,0 +1,8 @@
+---
+apiVersion: turtles-capi.cattle.io/v1alpha1
+kind: CAPIProvider
+metadata:
+  name: gcp
+  namespace: capg-system
+spec:
+  type: infrastructure
diff --git a/test/e2e/specs/import_gitops_mgmtv3.go b/test/e2e/specs/import_gitops_mgmtv3.go
@@ -93,7 +93,7 @@ type CreateMgmtV3UsingGitOpsSpecInput struct {
 // automatically imports into Rancher Manager.
 func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateMgmtV3UsingGitOpsSpecInput) {
 	var (
-		specName              = "creategitops-v3"
+		specName              = "creategitops"
 		input                 CreateMgmtV3UsingGitOpsSpecInput
 		namespace             *corev1.Namespace
 		repoName              string
@@ -172,6 +172,7 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM
 		input = inputGetter()
 		Expect(turtlesframework.Parse(&input)).To(Succeed(), "Failed to parse environment variables")
 
+		Expect(input.GitAddr).ToNot(BeEmpty(), "Invalid argument. input.GitAddr can't be empty when calling %s spec", specName)
 		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
 		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
 		Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)

diff --git a/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go b/test/e2e/suites/import-gitops-v3/import_gitops_v3_test.go
@@ -23,10 +23,12 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/komega"
 
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
 
 	"github.com/rancher/turtles/test/e2e"
 	"github.com/rancher/turtles/test/e2e/specs"
+	"github.com/rancher/turtles/test/testenv"
 )
 
 var _ = Describe("[Docker] [Kubeadm] - [management.cattle.io/v3] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel), func() {
@@ -36,6 +38,14 @@ var _ = Describe("[Docker] [Kubeadm] - [management.cattle.io/v3] Create and dele
 	})
 
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersYAML: [][]byte{
+				e2e.CapiProviders,
+			},
+			WaitForDeployments: testenv.DefaultDeployments,
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
@@ -65,6 +75,14 @@ var _ = Describe("[Docker] [RKE2] - [management.cattle.io/v3] Create and delete
 	})
 
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersYAML: [][]byte{
+				e2e.CapiProviders,
+			},
+			WaitForDeployments: testenv.DefaultDeployments,
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
@@ -93,12 +111,38 @@ var _ = Describe("[Azure] [AKS] - [management.cattle.io/v3] Create and delete CA
 		komega.SetContext(ctx)
 	})
 
+	AfterEach(func() {
+		testenv.CAPIOperatorRemoveProvider(ctx, testenv.CAPIOperatorRemoveProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProviders: []types.NamespacedName{{
+				Name:      "azure",
+				Namespace: "capz-system",
+			}},
+		})
+	})
+
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersSecretsYAML: [][]byte{
+				e2e.AzureIdentitySecret,
+			},
+			CAPIProvidersYAML: [][]byte{
+				e2e.AzureProvider,
+			},
+			WaitForDeployments: []testenv.NamespaceName{
+				{
+					Name:      "capz-controller-manager",
+					Namespace: "capz-system",
+				},
+			},
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
 			ClusterTemplate:                e2e.CAPIAzureAKSTopology,
-			ClusterName:                    "highlander-e2e-topology",
+			ClusterName:                    "cluster-aks",
 			ControlPlaneMachineCount:       ptr.To[int](1),
 			WorkerMachineCount:             ptr.To[int](1),
 			GitAddr:                        gitAddress,
@@ -120,12 +164,38 @@ var _ = Describe("[AWS] [EKS] - [management.cattle.io/v3] Create and delete CAPI
 		komega.SetContext(ctx)
 	})
 
+	AfterEach(func() {
+		testenv.CAPIOperatorRemoveProvider(ctx, testenv.CAPIOperatorRemoveProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProviders: []types.NamespacedName{{
+				Name:      "aws",
+				Namespace: "capa-system",
+			}},
+		})
+	})
+
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersSecretsYAML: [][]byte{
+				e2e.AWSProviderSecret,
+			},
+			CAPIProvidersYAML: [][]byte{
+				e2e.AWSProvider,
+			},
+			WaitForDeployments: []testenv.NamespaceName{
+				{
+					Name:      "capa-controller-manager",
+					Namespace: "capa-system",
+				},
+			},
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
 			ClusterTemplate:                e2e.CAPIAwsEKSMMP,
-			ClusterName:                    "clusterv3-eks",
+			ClusterName:                    "cluster-eks",
 			ControlPlaneMachineCount:       ptr.To[int](1),
 			WorkerMachineCount:             ptr.To[int](1),
 			GitAddr:                        gitAddress,
@@ -147,12 +217,38 @@ var _ = Describe("[GCP] [GKE] - [management.cattle.io/v3] Create and delete CAPI
 		komega.SetContext(ctx)
 	})
 
+	AfterEach(func() {
+		testenv.CAPIOperatorRemoveProvider(ctx, testenv.CAPIOperatorRemoveProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProviders: []types.NamespacedName{{
+				Name:      "gcp",
+				Namespace: "capg-system",
+			}},
+		})
+	})
+
 	specs.CreateMgmtV3UsingGitOpsSpec(ctx, func() specs.CreateMgmtV3UsingGitOpsSpecInput {
+		testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			CAPIProvidersSecretsYAML: [][]byte{
+				e2e.GCPProviderSecret,
+			},
+			CAPIProvidersYAML: [][]byte{
+				e2e.GCPProvider,
+			},
+			WaitForDeployments: []testenv.NamespaceName{
+				{
+					Name:      "capg-controller-manager",
+					Namespace: "capg-system",
+				},
+			},
+		})
+
 		return specs.CreateMgmtV3UsingGitOpsSpecInput{
 			E2EConfig:                      e2e.LoadE2EConfig(),
 			BootstrapClusterProxy:          bootstrapClusterProxy,
 			ClusterTemplate:                e2e.CAPIGCPGKE,
-			ClusterName:                    "clusterv3-gke",
+			ClusterName:                    "cluster-gke",
 			ControlPlaneMachineCount:       ptr.To[int](1),
 			WorkerMachineCount:             ptr.To[int](1),
 			GitAddr:                        gitAddress,

diff --git a/test/e2e/suites/import-gitops-v3/suite_test.go b/test/e2e/suites/import-gitops-v3/suite_test.go
@@ -21,13 +21,13 @@ package import_gitops_v3
 
 import (
 	"context"
-	"encoding/json"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/rancher/turtles/test/e2e"
 	"github.com/rancher/turtles/test/testenv"
+	"k8s.io/apimachinery/pkg/util/json"
 	capiframework "sigs.k8s.io/cluster-api/test/framework"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -73,50 +73,17 @@ var _ = SynchronizedBeforeSuite(
 
 		testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
 			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
-			CAPIProvidersYAML:     e2e.CapiProviders,
-			WaitForDeployments: append(
-				testenv.DefaultDeployments,
-				testenv.NamespaceName{
-					Name:      "caapf-controller-manager",
-					Namespace: e2e.RancherTurtlesNamespace,
-				}),
 		})
 
-		if !shortTestOnly() && !localTestOnly() {
-			By("Running full tests, deploying additional infrastructure providers")
-
-			testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
-				BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
-				CAPIProvidersSecretsYAML: [][]byte{
-					e2e.AWSProviderSecret,
-					e2e.AzureIdentitySecret,
-					e2e.GCPProviderSecret,
-				},
-				CAPIProvidersYAML: e2e.FullProviders,
-				WaitForDeployments: []testenv.NamespaceName{
-					{
-						Name:      "capa-controller-manager",
-						Namespace: "capa-system",
-					},
-					{
-						Name:      "capz-controller-manager",
-						Namespace: "capz-system",
-					},
-					{
-						Name:      "capg-controller-manager",
-						Namespace: "capg-system",
-					},
-				},
-			})
-		} else if Label(e2e.LocalTestLabel).MatchesLabelFilter(GinkgoLabelFilter()) {
+		if Label(e2e.LocalTestLabel).MatchesLabelFilter(GinkgoLabelFilter()) {
 			By("Running local vSphere tests, deploying vSphere infrastructure provider")
 
 			testenv.CAPIOperatorDeployProvider(ctx, testenv.CAPIOperatorDeployProviderInput{
 				BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
 				CAPIProvidersSecretsYAML: [][]byte{
 					e2e.VSphereProviderSecret,
 				},
-				CAPIProvidersYAML: e2e.CapvProvider,
+				CAPIProvidersYAML: [][]byte{e2e.CapvProvider},
 				WaitForDeployments: []testenv.NamespaceName{
 					{
 						Name:      "capv-controller-manager",
@@ -158,11 +125,11 @@ var _ = SynchronizedAfterSuite(
 	},
 	func() {
 		testenv.UninstallGitea(ctx, testenv.UninstallGiteaInput{
-			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
+			BootstrapClusterProxy: bootstrapClusterProxy,
 		})
 
 		testenv.UninstallRancherTurtles(ctx, testenv.UninstallRancherTurtlesInput{
-			BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
+			BootstrapClusterProxy: bootstrapClusterProxy,
 		})
 
 		testenv.CleanupTestCluster(ctx, testenv.CleanupTestClusterInput{