diff --git a/test/e2e/basic/api-group/enable_api_group_extentions.go b/test/e2e/basic/api-group/enable_api_group_extentions.go index 546d2f7212..0ce85c4983 100644 --- a/test/e2e/basic/api-group/enable_api_group_extentions.go +++ b/test/e2e/basic/api-group/enable_api_group_extentions.go @@ -39,12 +39,12 @@ func APIExtensionsVersionsTest() { label := "for=backup" srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml" BeforeEach(func() { - if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" { + if veleroCfg.ActiveClusterContext == "" && veleroCfg.StandbyClusterContext == "" { Skip("CRD with apiextension versions migration test needs 2 clusters") } veleroCfg = VeleroCfg - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) - srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.ActiveClusterContext)).To(Succeed()) + srcVersions, err := GetAPIVersions(veleroCfg.ActiveClient, resourceName) Expect(err).ShouldNot(HaveOccurred()) dstVersions, err := GetAPIVersions(veleroCfg.StandbyClient, resourceName) Expect(err).ShouldNot(HaveOccurred()) @@ -67,9 +67,9 @@ func APIExtensionsVersionsTest() { }) }) AfterEach(func() { - By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) - veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient + By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.ActiveClusterContext), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.ActiveClusterContext)).To(Succeed()) + veleroCfg.ClientToInstallVelero = veleroCfg.ActiveClient }) if CurrentSpecReport().Failed() && veleroCfg.FailFast { @@ -82,7 +82,7 @@ func APIExtensionsVersionsTest() { By("Uninstall Velero and delete CRD ", func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.ActiveClusterContext)).To(Succeed()) Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed()) @@ -98,14 +98,14 @@ func APIExtensionsVersionsTest() { backupName = "backup-" + UUIDgen.String() restoreName = "restore-" + UUIDgen.String() - By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) + By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.ActiveClusterContext), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.ActiveClusterContext)).To(Succeed()) veleroCfg.Features = "EnableAPIGroupVersions" veleroCfg.UseVolumeSnapshots = false Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) }) - By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultClusterContext), func() { + By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.ActiveClusterContext), func() { Expect(InstallCRD(context.Background(), srcCrdYaml)).To(Succeed()) Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed()) Expect(WaitForCRDEstablished(crdName)).To(Succeed()) diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index 15e92699e6..49b555ecf2 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -28,13 +28,13 @@ import ( v1 "k8s.io/api/core/v1" . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/common" . "github.com/vmware-tanzu/velero/test/util/k8s" ) type BackupVolumeInfo struct { - TestCase + BRCase SnapshotVolumes bool DefaultVolumesToFSBackup bool SnapshotMoveData bool @@ -42,7 +42,25 @@ type BackupVolumeInfo struct { } func (v *BackupVolumeInfo) Init() error { - v.TestCase.Init() + v.BRCase.Init() + + if v.VeleroCfg.CloudProvider == Vsphere && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { + fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) + Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") + } + + if strings.Contains(v.VeleroCfg.Features, FeatureCSI) { + if strings.Contains(v.CaseBaseName, "native-snapshot") { + fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) + Skip("Skip native snapshot case due to CSI feature is enabled.") + } + } else { + if strings.Contains(v.CaseBaseName, "csi") { + fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) + Skip("Skip CSI cases due to CSI feature is not enabled.") + } + } + v.CaseBaseName = v.CaseBaseName + v.UUIDgen v.BackupName = "backup-" + v.CaseBaseName v.RestoreName = "restore-" + v.CaseBaseName @@ -76,26 +94,6 @@ func (v *BackupVolumeInfo) Init() error { return nil } -func (v *BackupVolumeInfo) Start() error { - if v.VeleroCfg.CloudProvider == Vsphere && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { - fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) - Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") - } - - if strings.Contains(v.VeleroCfg.Features, FeatureCSI) { - if strings.Contains(v.CaseBaseName, "native-snapshot") { - fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) - Skip("Skip native snapshot case due to CSI feature is enabled.") - } - } else { - if strings.Contains(v.CaseBaseName, "csi") { - fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) - Skip("Skip CSI cases due to CSI feature is not enabled.") - } - } - v.TestCase.Start() - return nil -} func (v *BackupVolumeInfo) CreateResources() error { labels := map[string]string{ "volume-info": "true", @@ -147,7 +145,7 @@ func (v *BackupVolumeInfo) CreateResources() error { return nil } -func (v *BackupVolumeInfo) Destroy() error { +func (v *BackupVolumeInfo) DeleteResources() error { err := CleanupNamespaces(v.Ctx, v.Client, v.CaseBaseName) if err != nil { return errors.Wrap(err, "Could cleanup retrieve namespaces") diff --git a/test/e2e/basic/backup-volume-info/csi_data_mover.go b/test/e2e/basic/backup-volume-info/csi_data_mover.go index de1ca5c423..840d1a37ff 100644 --- a/test/e2e/basic/backup-volume-info/csi_data_mover.go +++ b/test/e2e/basic/backup-volume-info/csi_data_mover.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/providers" . "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -30,7 +30,7 @@ var CSIDataMoverVolumeInfoTest func() = TestFunc(&CSIDataMoverVolumeInfo{ BackupVolumeInfo{ SnapshotMoveData: true, SnapshotVolumes: true, - TestCase: TestCase{ + BRCase: BRCase{ CaseBaseName: "csi-data-mover-volumeinfo", TestMsg: &TestMSG{ Desc: "Test backup's VolumeInfo metadata content for CSI data mover case.", diff --git a/test/e2e/basic/backup-volume-info/csi_snapshot.go b/test/e2e/basic/backup-volume-info/csi_snapshot.go index ef476421c0..12efc0c40e 100644 --- a/test/e2e/basic/backup-volume-info/csi_snapshot.go +++ b/test/e2e/basic/backup-volume-info/csi_snapshot.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/providers" . "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -29,7 +29,7 @@ import ( var CSISnapshotVolumeInfoTest func() = TestFunc(&CSISnapshotVolumeInfo{ BackupVolumeInfo{ SnapshotVolumes: true, - TestCase: TestCase{ + BRCase: BRCase{ CaseBaseName: "csi-snapshot-volumeinfo", TestMsg: &TestMSG{ Desc: "Test backup's VolumeInfo metadata content for CSI snapshot case.", diff --git a/test/e2e/basic/backup-volume-info/filesystem_upload.go b/test/e2e/basic/backup-volume-info/filesystem_upload.go index d58eee5ada..ed7157ab72 100644 --- a/test/e2e/basic/backup-volume-info/filesystem_upload.go +++ b/test/e2e/basic/backup-volume-info/filesystem_upload.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/providers" . "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -29,7 +29,7 @@ import ( var FilesystemUploadVolumeInfoTest func() = TestFunc(&FilesystemUploadVolumeInfo{ BackupVolumeInfo{ DefaultVolumesToFSBackup: true, - TestCase: TestCase{ + BRCase: BRCase{ CaseBaseName: "fs-upload-volumeinfo", TestMsg: &TestMSG{ Desc: "Test backup's VolumeInfo metadata content for filesystem upload case.", diff --git a/test/e2e/basic/backup-volume-info/native_snapshot.go b/test/e2e/basic/backup-volume-info/native_snapshot.go index b4200f85ca..aef227c9b6 100644 --- a/test/e2e/basic/backup-volume-info/native_snapshot.go +++ b/test/e2e/basic/backup-volume-info/native_snapshot.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/providers" . "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -29,7 +29,7 @@ import ( var NativeSnapshotVolumeInfoTest func() = TestFunc(&NativeSnapshotVolumeInfo{ BackupVolumeInfo{ SnapshotVolumes: true, - TestCase: TestCase{ + BRCase: BRCase{ UseVolumeSnapshots: true, CaseBaseName: "native-snapshot-volumeinfo", TestMsg: &TestMSG{ diff --git a/test/e2e/basic/backup-volume-info/skipped_volumes.go b/test/e2e/basic/backup-volume-info/skipped_volumes.go index 0f1ccec0c2..43315d8c72 100644 --- a/test/e2e/basic/backup-volume-info/skipped_volumes.go +++ b/test/e2e/basic/backup-volume-info/skipped_volumes.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/providers" . "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -29,7 +29,7 @@ import ( var SkippedVolumeInfoTest func() = TestFunc(&SkippedVolumeInfo{ BackupVolumeInfo{ SnapshotVolumes: false, - TestCase: TestCase{ + BRCase: BRCase{ CaseBaseName: "skipped-volumes-volumeinfo", TestMsg: &TestMSG{ Desc: "Test backup's VolumeInfo metadata content for volume-skipped case.", diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index a2ebc24c8b..ef5eec36d4 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -8,26 +8,26 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/kibishii" ) type NamespaceMapping struct { - TestCase + BRCase MappedNamespaceList []string kibishiiData *KibishiiData } const NamespaceBaseName string = "ns-mp-" -var OneNamespaceMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 1, UseVolumeSnapshots: false}}) -var MultiNamespacesMappingResticTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 2, UseVolumeSnapshots: false}}) -var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 1, UseVolumeSnapshots: true}}) -var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NamespacesTotal: 2, UseVolumeSnapshots: true}}) +var OneNamespaceMappingResticTest func() = TestFunc(&NamespaceMapping{BRCase: BRCase{NamespacesTotal: 1, UseVolumeSnapshots: false}}) +var MultiNamespacesMappingResticTest func() = TestFunc(&NamespaceMapping{BRCase: BRCase{NamespacesTotal: 2, UseVolumeSnapshots: false}}) +var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{BRCase: BRCase{NamespacesTotal: 1, UseVolumeSnapshots: true}}) +var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{BRCase: BRCase{NamespacesTotal: 2, UseVolumeSnapshots: true}}) func (n *NamespaceMapping) Init() error { - n.TestCase.Init() + n.BRCase.Init() n.CaseBaseName = "ns-mp-" + n.UUIDgen n.BackupName = "backup-" + n.CaseBaseName n.RestoreName = "restore-" + n.CaseBaseName @@ -128,7 +128,7 @@ func (n *NamespaceMapping) Clean() error { } } - return n.GetTestCase().Clean() + return n.BRCase.Clean() } return nil diff --git a/test/e2e/basic/nodeport.go b/test/e2e/basic/nodeport.go index 7cee801a08..d347b7c1f8 100644 --- a/test/e2e/basic/nodeport.go +++ b/test/e2e/basic/nodeport.go @@ -11,13 +11,14 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - . "github.com/vmware-tanzu/velero/test/e2e/test" + "github.com/vmware-tanzu/velero/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/velero" ) type NodePort struct { - TestCase + BRCase labels map[string]string serviceName string namespaceToCollision []string @@ -29,7 +30,12 @@ const NodeportBaseName string = "nodeport-" var NodePortTest func() = TestFunc(&NodePort{}) func (n *NodePort) Init() error { - n.TestCase.Init() + n.BRCase.Init() + + if n.VeleroCfg.CloudProvider == test.Azure || n.VeleroCfg.CloudProvider == test.AWS { + Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") + } + n.CaseBaseName = NodeportBaseName + n.UUIDgen n.BackupName = "backup-" + n.CaseBaseName n.RestoreName = "restore-" + n.CaseBaseName @@ -77,7 +83,7 @@ func (n *NodePort) CreateResources() error { return nil } -func (n *NodePort) Destroy() error { +func (n *NodePort) DeleteResources() error { for i, ns := range *n.NSIncluded { By(fmt.Sprintf("Start to destroy namespace %s......", n.CaseBaseName), func() { Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, NodeportBaseName)).To(Succeed(), diff --git a/test/e2e/basic/pvc-selected-node-changing.go b/test/e2e/basic/pvc-selected-node-changing.go index e200929c11..89f5245e81 100644 --- a/test/e2e/basic/pvc-selected-node-changing.go +++ b/test/e2e/basic/pvc-selected-node-changing.go @@ -9,13 +9,13 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/velero" ) type PVCSelectedNodeChanging struct { - TestCase + BRCase labels map[string]string data map[string]string configmaptName string @@ -32,7 +32,7 @@ type PVCSelectedNodeChanging struct { var PVCSelectedNodeChangingTest func() = TestFunc(&PVCSelectedNodeChanging{}) func (p *PVCSelectedNodeChanging) Init() error { - p.TestCase.Init() + p.BRCase.Init() p.CaseBaseName = "psnc-" + p.UUIDgen p.namespace = p.CaseBaseName p.mappedNS = p.namespace + "-mapped" @@ -104,7 +104,7 @@ func (p *PVCSelectedNodeChanging) CreateResources() error { return nil } -func (p *PVCSelectedNodeChanging) Destroy() error { +func (p *PVCSelectedNodeChanging) DeleteResources() error { By(fmt.Sprintf("Start to destroy namespace %s......", p.CaseBaseName), func() { Expect(CleanupNamespacesWithPoll(p.Ctx, p.Client, p.CaseBaseName)).To(Succeed(), fmt.Sprintf("Failed to delete namespace %s", p.CaseBaseName)) @@ -144,7 +144,7 @@ func (p *PVCSelectedNodeChanging) Clean() error { if CurrentSpecReport().Failed() && p.VeleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { - p.TestCase.Clean() + p.BRCase.Clean() By(fmt.Sprintf("Clean namespace with prefix %s after test", p.mappedNS), func() { CleanupNamespaces(p.Ctx, p.Client, p.mappedNS) }) diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index b99154a563..c476e62ea7 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -25,19 +25,19 @@ import ( "github.com/pkg/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) type MultiNSBackup struct { - TestCase + BRCase IsScalTest bool NSExcluded *[]string TimeoutDuration time.Duration } func (m *MultiNSBackup) Init() error { - m.TestCase.Init() + m.BRCase.Init() m.CaseBaseName = "nstest-" + m.UUIDgen m.BackupName = "backup-" + m.CaseBaseName m.RestoreName = "restore-" + m.CaseBaseName @@ -111,7 +111,7 @@ func (m *MultiNSBackup) Verify() error { return nil } -func (m *MultiNSBackup) Destroy() error { +func (m *MultiNSBackup) DeleteResources() error { err := CleanupNamespaces(m.Ctx, m.Client, m.CaseBaseName) if err != nil { return errors.Wrap(err, "Could cleanup retrieve namespaces") diff --git a/test/e2e/basic/resources-check/namespaces_annotation.go b/test/e2e/basic/resources-check/namespaces_annotation.go index e698d48185..d3d40a75e3 100644 --- a/test/e2e/basic/resources-check/namespaces_annotation.go +++ b/test/e2e/basic/resources-check/namespaces_annotation.go @@ -22,16 +22,16 @@ import ( "github.com/pkg/errors" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) type NSAnnotationCase struct { - TestCase + BRCase } func (n *NSAnnotationCase) Init() error { - n.TestCase.Init() + n.BRCase.Init() n.CaseBaseName = "namespace-annotations-" + n.UUIDgen n.BackupName = "backup-" + n.CaseBaseName n.RestoreName = "restore-" + n.CaseBaseName diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index b79c3615f4..2846cd6180 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -14,22 +14,6 @@ See the License for the specific language governing permissions and limitations under the Licensm. */ -/* -Copyright 2021 the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package basic import ( @@ -39,16 +23,16 @@ import ( . "github.com/onsi/ginkgo/v2" "github.com/pkg/errors" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) type RBACCase struct { - TestCase + BRCase } func (r *RBACCase) Init() error { - r.TestCase.Init() + r.BRCase.Init() r.CaseBaseName = "rabc-" + r.UUIDgen r.BackupName = "backup-" + r.CaseBaseName r.RestoreName = "restore-" + r.CaseBaseName @@ -155,7 +139,7 @@ func (r *RBACCase) Verify() error { return nil } -func (r *RBACCase) Destroy() error { +func (r *RBACCase) DeleteResources() error { //cleanup clusterrole err := CleanupClusterRole(r.Ctx, r.Client, r.CaseBaseName) if err != nil { @@ -180,7 +164,7 @@ func (r *RBACCase) Clean() error { if CurrentSpecReport().Failed() && r.VeleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { - return r.Destroy() + return r.DeleteResources() } return nil diff --git a/test/e2e/basic/resources-check/resources_check.go b/test/e2e/basic/resources-check/resources_check.go index 9db5198bc6..443fc56eba 100644 --- a/test/e2e/basic/resources-check/resources_check.go +++ b/test/e2e/basic/resources-check/resources_check.go @@ -33,11 +33,11 @@ limitations under the License. package basic import ( - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" ) -func GetResourcesCheckTestCases() []VeleroBackupRestoreTest { - return []VeleroBackupRestoreTest{ +func GetResourcesCheckTestCases() []BackupRestoreTest { + return []BackupRestoreTest{ &NSAnnotationCase{}, &MultiNSBackup{IsScalTest: false}, &RBACCase{}, diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index e2e5a02967..5dbe5b5315 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -9,13 +9,13 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/velero" ) type StorageClasssChanging struct { - TestCase + BRCase labels map[string]string data map[string]string cmName string @@ -34,7 +34,7 @@ const SCCBaseName string = "scc-" var StorageClasssChangingTest func() = TestFunc(&StorageClasssChanging{}) func (s *StorageClasssChanging) Init() error { - s.TestCase.Init() + s.BRCase.Init() s.CaseBaseName = SCCBaseName + s.UUIDgen s.namespace = s.CaseBaseName s.BackupName = "backup-" + s.CaseBaseName @@ -97,7 +97,7 @@ func (s *StorageClasssChanging) CreateResources() error { return nil } -func (s *StorageClasssChanging) Destroy() error { +func (s *StorageClasssChanging) DeleteResources() error { By(fmt.Sprintf("Expect storage class of PV %s to be %s ", s.volume, s.srcStorageClass), func() { pvName, err := GetPVByPVCName(s.Client, s.namespace, s.pvcName) Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by PVC name %s", s.pvcName)) @@ -146,7 +146,7 @@ func (s *StorageClasssChanging) Clean() error { fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName)) }) DeleteConfigMap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.cmName) - s.TestCase.Clean() + s.BRCase.Clean() } return nil diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 51584d85ba..22b4e46b88 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -78,7 +78,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { By("Clean backups after test", func() { - veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient + veleroCfg.ClientToInstallVelero = veleroCfg.ActiveClient DeleteAllBackups(context.Background(), &veleroCfg) }) By(fmt.Sprintf("Delete sample workload namespace %s", bslDeletionTestNs), func() { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 68de2ecc46..e954baa77a 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -236,7 +236,7 @@ func init() { "frequency of garbage collection.", ) flag.StringVar( - &test.VeleroCfg.DefaultClusterContext, + &test.VeleroCfg.ActiveClusterContext, "default-cluster-context", "", "default cluster's kube config context, it's for migration test.", @@ -284,7 +284,7 @@ func init() { "plugins provider for standby cluster.", ) flag.StringVar( - &test.VeleroCfg.StandbyClusterObjectStoreProvider, + &test.VeleroCfg.StandbyObjectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.", @@ -302,7 +302,7 @@ func init() { "a switch for disable informer cache.", ) flag.StringVar( - &test.VeleroCfg.DefaultClusterName, + &test.VeleroCfg.ActiveClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.", @@ -320,7 +320,7 @@ func init() { "EKS plicy ARN for creating AWS IAM service account.", ) flag.StringVar( - &test.VeleroCfg.DefaultCLSServiceAccountName, + &test.VeleroCfg.ActiveCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.", @@ -631,17 +631,17 @@ var _ = Describe( func GetKubeConfigContext() error { var err error var tcDefault, tcStandby k8s.TestClient - tcDefault, err = k8s.NewTestClient(test.VeleroCfg.DefaultClusterContext) - test.VeleroCfg.DefaultClient = &tcDefault - test.VeleroCfg.ClientToInstallVelero = test.VeleroCfg.DefaultClient - test.VeleroCfg.ClusterToInstallVelero = test.VeleroCfg.DefaultClusterName - test.VeleroCfg.ServiceAccountNameToInstall = test.VeleroCfg.DefaultCLSServiceAccountName + tcDefault, err = k8s.NewTestClient(test.VeleroCfg.ActiveClusterContext) + test.VeleroCfg.ActiveClient = &tcDefault + test.VeleroCfg.ClientToInstallVelero = test.VeleroCfg.ActiveClient + test.VeleroCfg.ClusterToInstallVelero = test.VeleroCfg.ActiveClusterName + test.VeleroCfg.ServiceAccountNameToInstall = test.VeleroCfg.ActiveCLSServiceAccountName if err != nil { return err } - if test.VeleroCfg.DefaultClusterContext != "" { - err = k8s.KubectlConfigUseContext(context.Background(), test.VeleroCfg.DefaultClusterContext) + if test.VeleroCfg.ActiveClusterContext != "" { + err = k8s.KubectlConfigUseContext(context.Background(), test.VeleroCfg.ActiveClusterContext) if err != nil { return err } @@ -659,8 +659,6 @@ func GetKubeConfigContext() error { return nil } -var testSuitePassed bool - func TestE2e(t *testing.T) { // Skip running E2E tests when running only "short" tests because: // 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations. @@ -669,26 +667,49 @@ func TestE2e(t *testing.T) { t.Skip("Skipping E2E tests") } - if !slices.Contains(test.LocalCloudProviders, test.VeleroCfg.CloudProvider) { - fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider") + var err error + test.VeleroCfg.ObjectStoreProvider, err = fillAndValidateObjectStoreProvider( + test.VeleroCfg.CloudProvider, + test.VeleroCfg.ObjectStoreProvider, + ) + if err != nil { + t.Error(err) + } + + test.VeleroCfg.StandbyObjectStoreProvider, err = fillAndValidateObjectStoreProvider( + test.VeleroCfg.StandbyClusterCloudProvider, + test.VeleroCfg.StandbyObjectStoreProvider, + ) + if err != nil { + t.Error(err) + } + + if slices.Contains(test.CloudProviders, test.VeleroCfg.CloudProvider) { // If ObjectStoreProvider is not provided, then using the value same as CloudProvider if test.VeleroCfg.ObjectStoreProvider == "" { + fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider") test.VeleroCfg.ObjectStoreProvider = test.VeleroCfg.CloudProvider } + + if test.VeleroCfg.CloudProvider == test.Vsphere { + test.VeleroCfg.ObjectStoreProvider = test.AWS + } } else { if test.VeleroCfg.ObjectStoreProvider == "" { t.Error(errors.New("No object store provider specified - must be specified when using kind as the cloud provider")) // Must have an object store provider } } + if slices.Contains(test.CloudProviders, test.VeleroCfg.StandbyClusterCloudProvider) { - var err error - if err = GetKubeConfigContext(); err != nil { + } + + if err := GetKubeConfigContext(); err != nil { fmt.Println(err) t.FailNow() } RegisterFailHandler(Fail) - testSuitePassed = RunSpecs(t, "E2e Suite") + RunSpecs(t, "E2e Suite") } var _ = BeforeSuite(func() { @@ -719,40 +740,65 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) + // If the Velero is installed during test, and the FailFast is not enabled, + // uninstall Velero. If not, either Velero is not installed, or kept it for debug. + if !test.InstallVelero || test.VeleroCfg.FailFast { + return + } + + // ctx is used to uninstall Velero on both Active and Standby cluster, + // enlarge time to 8 minutes. + ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*8) defer ctxCancel() - By("Delete StorageClasses created by E2E") - Expect( - k8s.DeleteStorageClass( + // Uninstall Velero on Active cluster. + if test.VeleroCfg.ActiveClusterContext != "" { + By("Uninstall Velero on Active cluster") + Expect(veleroutil.DeleteVeleroAndRelatedResources( ctx, - *test.VeleroCfg.ClientToInstallVelero, - test.StorageClassName, - ), - ).To(Succeed()) - Expect( - k8s.DeleteStorageClass( - ctx, - *test.VeleroCfg.ClientToInstallVelero, - test.StorageClassName2, - ), - ).To(Succeed()) - - if strings.EqualFold(test.VeleroCfg.Features, test.FeatureCSI) && - test.VeleroCfg.UseVolumeSnapshots { - By("Delete VolumeSnapshotClass created by E2E") - Expect( - k8s.KubectlDeleteByFile( - ctx, - fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", test.VeleroCfg.CloudProvider), - ), - ).To(Succeed()) + test.VeleroCfg.ActiveClusterContext, + test.VeleroCfg.ActiveClient, + test.VeleroCfg.ProviderName, + )).To(Succeed()) } - // If the Velero is installed during test, and the FailFast is not enabled, - // uninstall Velero. If not, either Velero is not installed, or kept it for debug on failure. - if test.InstallVelero && (testSuitePassed || !test.VeleroCfg.FailFast) { - By("release test resources after testing") - Expect(veleroutil.VeleroUninstall(ctx, test.VeleroCfg)).To(Succeed()) + // Uninstall Velero on Standby cluster. + if test.VeleroCfg.StandbyClusterContext != "" { + By("Uninstall Velero on Standby cluster") + Expect(veleroutil.DeleteVeleroAndRelatedResources( + ctx, + test.VeleroCfg.StandbyClusterContext, + test.VeleroCfg.StandbyClient, + test.VeleroCfg.StandbyClusterCloudProvider, + )).To(Succeed()) } + + // Switch the k8s context back to Active cluster. + Expect(k8s.KubectlConfigUseContext(ctx, test.VeleroCfg.ActiveClusterContext)).To(Succeed()) }) + +func fillAndValidateObjectStoreProvider( + cloudProvider string, + objectStoreProvider string, +) (string, error) { + result := objectStoreProvider + if slices.Contains(test.CloudProviders, cloudProvider) { + // If ObjectStoreProvider is not provided, then using the value same as CloudProvider + if objectStoreProvider == "" { + fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider") + result = cloudProvider + } + + if cloudProvider == test.Vsphere { + fmt.Println("For vSphere platform, use AWS plugin as the object store provider.") + result = test.AWS + } + } else { + if objectStoreProvider == "" { + // Must have an object store provider + return result, errors.New("No object store provider specified - must be specified when using kind as the cloud provider") + } + } + + return result, nil +} diff --git a/test/e2e/framework/backup-restore.go b/test/e2e/framework/backup-restore.go new file mode 100644 index 0000000000..be48e66642 --- /dev/null +++ b/test/e2e/framework/backup-restore.go @@ -0,0 +1,280 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "crypto/rand" + "fmt" + "math/big" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerotest "github.com/vmware-tanzu/velero/test" + k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" + veleroutil "github.com/vmware-tanzu/velero/test/util/velero" +) + +/* +The BackupRestoreTest interface is just could be suit for the cases that follow the test flow of +create resources, backup, delete test resource, restore and verify. +And the cases have similar execute function and similar data. +*/ +type BackupRestoreTest interface { + Init() error + InstallVelero() error + CreateResources() error + Backup() error + DeleteResources() error + Restore() error + Verify() error + Clean() error + GetTestMsg() *TestMSG +} + +type TestMSG struct { + Desc string + Text string + FailedMSG string +} + +type BRCase struct { + BackupName string + RestoreName string + CaseBaseName string + BackupArgs []string + RestoreArgs []string + NamespacesTotal int + TestMsg *TestMSG + Client k8sutil.TestClient + NSIncluded *[]string + UseVolumeSnapshots bool + VeleroCfg velerotest.VeleroConfig + RestorePhaseExpect velerov1api.RestorePhase + Ctx context.Context + CtxCancel context.CancelFunc + UUIDgen string +} + +func TestFunc(test BackupRestoreTest) func() { + return func() { + TestIt(test) + } +} + +func TestFuncWithMultiIt(tests []BackupRestoreTest) func() { + return func() { + for k := range tests { + TestIt(tests[k]) + } + } +} + +func TestIt(test BackupRestoreTest) { + It("Run BackupRestore E2E test case", func() { + Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) + }) +} + +func (br *BRCase) Init() error { + br.Ctx, br.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) + defer br.CtxCancel() + + br.UUIDgen = GenerateUUID() + br.VeleroCfg = velerotest.VeleroCfg + br.Client = *br.VeleroCfg.ClientToInstallVelero + return nil +} + +func (br *BRCase) InstallVelero() error { + if velerotest.InstallVelero { + return veleroutil.PrepareVelero(br.Ctx, br.CaseBaseName, br.VeleroCfg) + } + + return nil +} + +func (br *BRCase) CreateResources() error { + return nil +} + +func (br *BRCase) Backup() error { + By("Start to backup ......", func() { + br.BackupName = "backup-" + br.CaseBaseName + + Expect( + veleroutil.VeleroBackupExec( + br.Ctx, + br.VeleroCfg.VeleroCLI, + br.VeleroCfg.VeleroNamespace, + br.BackupName, + br.BackupArgs, + )).To( + Succeed(), + func() string { + veleroutil.RunDebug( + br.Ctx, + br.VeleroCfg.VeleroCLI, + br.VeleroCfg.VeleroNamespace, + br.BackupName, + "", + ) + + return "Failed to backup resources" + }) + }) + + return nil +} + +func (br *BRCase) DeleteResources() error { + By(fmt.Sprintf("Start to delete namespace %s......", br.CaseBaseName), func() { + Expect(k8sutil.CleanupNamespacesWithPoll(br.Ctx, br.Client, br.CaseBaseName)).To(Succeed(), "Could cleanup retrieve namespaces") + }) + return nil +} + +func (br *BRCase) Restore() error { + if len(br.RestoreArgs) == 0 { + return nil + } + + br.RestoreName = "restore-" + br.CaseBaseName + + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + By("Waiting 5 minutes to make sure the snapshots are ready...", func() { + if br.UseVolumeSnapshots && br.VeleroCfg.CloudProvider != velerotest.Vsphere { + time.Sleep(5 * time.Minute) + } + }) + + By("Start to restore ......", func() { + if br.RestorePhaseExpect == "" { + br.RestorePhaseExpect = velerov1api.RestorePhaseCompleted + } + Expect( + veleroutil.VeleroRestoreExec( + br.Ctx, + br.VeleroCfg.VeleroCLI, + br.VeleroCfg.VeleroNamespace, + br.RestoreName, + br.RestoreArgs, + br.RestorePhaseExpect, + )).To( + Succeed(), + func() string { + veleroutil.RunDebug( + br.Ctx, + br.VeleroCfg.VeleroCLI, + br.VeleroCfg.VeleroNamespace, + "", + br.RestoreName, + ) + + return "Fail to restore workload" + }) + }) + return nil +} + +func (br *BRCase) Verify() error { + return nil +} + +func (br *BRCase) Clean() (err error) { + veleroCfg := br.VeleroCfg + if CurrentSpecReport().Failed() && veleroCfg.FailFast { + fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") + } else { + By(fmt.Sprintf("Clean namespace with prefix %s after test", br.CaseBaseName), func() { + if err = k8sutil.CleanupNamespaces(br.Ctx, br.Client, br.CaseBaseName); err != nil { + fmt.Println("Fail to cleanup namespaces: ", err) + } + }) + By("Clean backups after test", func() { + veleroCfg.ClientToInstallVelero = &br.Client + if err = veleroutil.DeleteAllBackups(br.Ctx, &veleroCfg); err != nil { + fmt.Println("Fail to clean backups after test: ", err) + } + }) + } + + return err +} + +func (br *BRCase) GetTestMsg() *TestMSG { + return br.TestMsg +} + +func RunTestCase(test BackupRestoreTest) error { + if test == nil { + return errors.New("No case should be tested") + } + + fmt.Println("Start to run case: ", test.GetTestMsg().Text) + Expect(test.Init()).To(Succeed()) + + defer Expect(test.Clean()).To(Succeed()) + + fmt.Println("Install Velero: ", test.GetTestMsg().Text) + Expect(test.InstallVelero()).To(Succeed()) + + fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) + + fmt.Printf("CreateResources %s\n", time.Now().Format("2006-01-02 15:04:05")) + err := test.CreateResources() + if err != nil { + return err + } + fmt.Printf("Backup %s\n", time.Now().Format("2006-01-02 15:04:05")) + err = test.Backup() + if err != nil { + return err + } + fmt.Printf("Destroy %s\n", time.Now().Format("2006-01-02 15:04:05")) + err = test.DeleteResources() + if err != nil { + return err + } + fmt.Printf("Restore %s\n", time.Now().Format("2006-01-02 15:04:05")) + err = test.Restore() + if err != nil { + return err + } + fmt.Printf("Verify %s\n", time.Now().Format("2006-01-02 15:04:05")) + err = test.Verify() + if err != nil { + return err + } + fmt.Printf("Finish run test %s\n", time.Now().Format("2006-01-02 15:04:05")) + return nil +} + +func GenerateUUID() string { + n, err := rand.Int(rand.Reader, big.NewInt(100000000)) + if err != nil { + fmt.Println("Fail to generate the random number: ", err) + return "" + } + return fmt.Sprintf("%08d", n) +} diff --git a/test/e2e/framework/migration.go b/test/e2e/framework/migration.go new file mode 100644 index 0000000000..ad564cecef --- /dev/null +++ b/test/e2e/framework/migration.go @@ -0,0 +1,313 @@ +package framework + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + "github.com/vmware-tanzu/velero/test" + velerotest "github.com/vmware-tanzu/velero/test" + k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" + "github.com/vmware-tanzu/velero/test/util/kibishii" + veleroutil "github.com/vmware-tanzu/velero/test/util/velero" +) + +/* +MigrationTest interface is specific for the migration test scenario. +*/ +type MigrationTest interface { + Init() error + InstallVeleroOnActiveCluster() error + InstallVeleroOnStandbyCluster() error + CreateResourcesOnActiveCluster() error + BackupOnActiveCluster() error + RestoreOnStandbyCluster() error + VerifyOnStandbyCluster() error + CleanOnBothCluster() error + GetTestMsg() *TestMSG +} + +type MigrationCase struct { + BackupName string + RestoreName string + CaseBaseName string + BackupArgs []string + RestoreArgs []string + TestMsg *TestMSG + UseVolumeSnapshots bool + VeleroCfg velerotest.VeleroConfig + Ctx context.Context + CtxCancel context.CancelFunc + UUIDgen string + VeleroCLI2Version test.VeleroCLI2Version + KibishiiData kibishii.KibishiiData +} + +func (m *MigrationCase) Init() error { + m.Ctx, m.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) + defer m.CtxCancel() + + m.UUIDgen = GenerateUUID() + m.VeleroCfg = velerotest.VeleroCfg + + m.CaseBaseName = "migration-" + m.UUIDgen + + m.KibishiiData = *kibishii.DefaultKibishiiData + m.KibishiiData.ExpectedNodes = 3 + + // Message output by ginkgo + m.TestMsg = &TestMSG{ + Desc: "Test migration workload on two clusters", + FailedMSG: "Fail to test migrate between two clusters", + Text: "Test back up on default cluster, restore on standby cluster", + } + + return nil +} + +func (m *MigrationCase) InstallVeleroOnActiveCluster() error { + if m.VeleroCLI2Version.VeleroCLI == "" { + //Assume tag of velero server image is identical to velero CLI version + //Download velero CLI if it's empty according to velero CLI version + By( + fmt.Sprintf("Install the expected version Velero CLI %s", + m.VeleroCLI2Version.VeleroVersion), + func() { + var err error + // "self" represents 1.14.x and future versions + if m.VeleroCLI2Version.VeleroVersion == "self" { + m.VeleroCLI2Version.VeleroCLI = m.VeleroCfg.VeleroCLI + } else { + m.VeleroCfg, err = veleroutil.SetImagesToDefaultValues( // TODO + m.VeleroCfg, + m.VeleroCLI2Version.VeleroVersion, + ) + Expect(err).To(Succeed(), + "Fail to set images for the migrate-from Velero installation.") + + m.VeleroCLI2Version.VeleroCLI, err = veleroutil.InstallVeleroCLI( + m.VeleroCLI2Version.VeleroVersion) + Expect(err).To(Succeed()) + } + }, + ) + } + + By(fmt.Sprintf("Install Velero on default cluster (%s)", m.VeleroCfg.ActiveClusterContext), + func() { + Expect(k8sutil.KubectlConfigUseContext( + m.Ctx, m.VeleroCfg.ActiveClusterContext)).To(Succeed()) + m.VeleroCfg.MigrateFromVeleroVersion = m.VeleroCLI2Version.VeleroVersion + m.VeleroCfg.VeleroCLI = m.VeleroCLI2Version.VeleroCLI + m.VeleroCfg.ClientToInstallVelero = m.VeleroCfg.ActiveClient + m.VeleroCfg.ClusterToInstallVelero = m.VeleroCfg.ActiveClusterName + m.VeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.ActiveCLSServiceAccountName + m.VeleroCfg.UseVolumeSnapshots = m.UseVolumeSnapshots + m.VeleroCfg.UseNodeAgent = !m.UseVolumeSnapshots + + version, err := veleroutil.GetVeleroVersion(m.Ctx, m.VeleroCLI2Version.VeleroVersion, true) + Expect(err).To(Succeed(), "Fail to get Velero version") + m.VeleroCfg.VeleroVersion = version + + Expect(veleroutil.VeleroInstall(m.Ctx, &m.VeleroCfg, false)).To(Succeed()) + if m.VeleroCLI2Version.VeleroVersion != "self" { + Expect(veleroutil.CheckVeleroVersion( + m.Ctx, + m.VeleroCfg.VeleroCLI, // TODO + m.VeleroCfg.MigrateFromVeleroVersion, // TODO + )).To(Succeed()) + } + }, + ) + + return nil +} + +func (m *MigrationCase) InstallVeleroOnStandbyCluster() error { + return nil +} + +func (m *MigrationCase) CreateResourcesOnActiveCluster() error { + By("Create namespace for sample workload", func() { + Expect(k8sutil.CreateNamespace( + m.Ctx, + *m.VeleroCfg.ActiveClient, + m.CaseBaseName, + )).To(Succeed(), + fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", + m.CaseBaseName)) + }) + + By("Deploy sample workload of Kibishii", func() { + Expect(kibishii.KibishiiPrepareBeforeBackup( + m.Ctx, + *m.VeleroCfg.ActiveClient, + m.VeleroCfg.CloudProvider, + m.CaseBaseName, + m.VeleroCfg.RegistryCredentialFile, + m.VeleroCfg.Features, + m.VeleroCfg.KibishiiDirectory, + m.VeleroCfg.UseVolumeSnapshots, + &m.KibishiiData, + )).To(Succeed()) + }) + + return nil +} + +func (m *MigrationCase) BackupOnActiveCluster() error { + By(fmt.Sprintf("Backup namespace %s", m.CaseBaseName), func() { + m.BackupArgs = []string{ + "create", "--namespace", m.VeleroCfg.VeleroNamespace, + "backup", m.BackupName, + "--include-namespaces", m.CaseBaseName, + "--wait", + } + + if m.UseVolumeSnapshots { + m.BackupArgs = append(m.BackupArgs, "--snapshot-volumes=true") + } else { + m.BackupArgs = append(m.BackupArgs, "--default-volumes-to-fs-backup") + } + + if m.VeleroCfg.SnapshotMoveData { + m.BackupArgs = append(m.BackupArgs, "--snapshot-move-data") + } + + Expect(veleroutil.VeleroBackupExec( + m.Ctx, + m.VeleroCfg.VeleroCLI, // TODO + m.VeleroCfg.VeleroNamespace, + m.BackupName, + m.BackupArgs, + )).To(Succeed(), func() string { + veleroutil.RunDebug( + context.Background(), + m.VeleroCfg.VeleroCLI, // TODO + m.VeleroCfg.VeleroNamespace, + m.BackupName, + "", + ) + return "Failed to backup resources" + }) + }) + + return nil +} + +func (m *MigrationCase) RestoreOnStandbyCluster() error { + return nil +} + +func (m *MigrationCase) VerifyOnStandbyCluster() error { + return nil +} + +func (m *MigrationCase) CleanOnBothCluster() error { + By("Clean resource on active cluster.", func() { + Expect(veleroutil.DeleteVeleroAndRelatedResources( + m.Ctx, + m.VeleroCfg.ActiveClusterContext, + m.VeleroCfg.ActiveClient, + m.VeleroCfg.ProviderName, + )).To(Succeed()) + + Expect( + k8sutil.DeleteNamespace( + m.Ctx, + *m.VeleroCfg.ActiveClient, + m.CaseBaseName, + true, + ), + ).To(Succeed()) + }) + + By("Clean resource on standby cluster.", func() { + Expect(veleroutil.DeleteVeleroAndRelatedResources( + m.Ctx, + m.VeleroCfg.StandbyClusterContext, + m.VeleroCfg.StandbyClient, + m.VeleroCfg.StandbyClusterCloudProvider, + )).To(Succeed()) + + Expect( + k8sutil.DeleteNamespace( + m.Ctx, + *m.VeleroCfg.StandbyClient, + m.CaseBaseName, + true, + ), + ).To(Succeed()) + }) + + By("Switch to default KubeConfig context", func() { + Expect(k8sutil.KubectlConfigUseContext( + m.Ctx, + m.VeleroCfg.ActiveClusterContext, + )).To(Succeed()) + }) + + return nil +} + +func (m *MigrationCase) GetTestMsg() *TestMSG { + return m.TestMsg +} + +func MigrationTestFunc(test MigrationTest) { + It("Run migration E2E test case", func() { + Expect(RunMigrationTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) + }) +} + +func RunMigrationTestCase(test MigrationTest) error { + if test == nil { + return errors.New("No case should be tested") + } + + if err := test.Init(); err != nil { + fmt.Println("Fail to init the migration test: ", err) + return err + } + + if err := test.InstallVeleroOnActiveCluster(); err != nil { + fmt.Println("Fail to install Velero on active cluster: ", err) + return err + } + + if err := test.InstallVeleroOnStandbyCluster(); err != nil { + fmt.Println("Fail to install Velero on standby cluster: ", err) + return err + } + + if err := test.CreateResourcesOnActiveCluster(); err != nil { + fmt.Println("Fail to create resources on active cluster: ", err) + return err + } + + if err := test.BackupOnActiveCluster(); err != nil { + fmt.Println("Fail to run backup on active cluster: ", err) + return err + } + + if err := test.RestoreOnStandbyCluster(); err != nil { + fmt.Println("Fail to run restore on standby cluster: ", err) + return err + } + + if err := test.VerifyOnStandbyCluster(); err != nil { + fmt.Println("Fail to verify the restore result on standby cluster: ", err) + return err + } + + if err := test.CleanOnBothCluster(); err != nil { + fmt.Println("Fail to clean the resources on both clusters: ", err) + return err + } + + return nil +} diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 7eb57daaec..4889eb9a9d 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -25,7 +25,7 @@ import ( . "github.com/onsi/gomega" "github.com/vmware-tanzu/velero/test" - framework "github.com/vmware-tanzu/velero/test/e2e/test" + "github.com/vmware-tanzu/velero/test/e2e/framework" util "github.com/vmware-tanzu/velero/test/util/csi" k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" "github.com/vmware-tanzu/velero/test/util/kibishii" @@ -34,22 +34,21 @@ import ( ) type migrationE2E struct { - framework.TestCase - useVolumeSnapshots bool - veleroCLI2Version test.VeleroCLI2Version - kibishiiData kibishii.KibishiiData + framework.MigrationCase } func MigrationWithSnapshots() { for _, veleroCLI2Version := range veleroutil.GetVersionList( test.VeleroCfg.MigrateFromVeleroCLI, test.VeleroCfg.MigrateFromVeleroVersion) { - framework.TestFunc( + framework.MigrationTestFunc( &migrationE2E{ - useVolumeSnapshots: true, - veleroCLI2Version: veleroCLI2Version, + framework.MigrationCase{ + UseVolumeSnapshots: true, + VeleroCLI2Version: veleroCLI2Version, + }, }, - )() + ) } } @@ -57,61 +56,36 @@ func MigrationWithFS() { for _, veleroCLI2Version := range veleroutil.GetVersionList( test.VeleroCfg.MigrateFromVeleroCLI, test.VeleroCfg.MigrateFromVeleroVersion) { - framework.TestFunc( + framework.MigrationTestFunc( &migrationE2E{ - useVolumeSnapshots: false, - veleroCLI2Version: veleroCLI2Version, + framework.MigrationCase{ + UseVolumeSnapshots: false, + VeleroCLI2Version: veleroCLI2Version, + }, }, - )() + ) } } func (m *migrationE2E) Init() error { By("Call the base E2E init", func() { - Expect(m.TestCase.Init()).To(Succeed()) + Expect(m.MigrationCase.Init()).To(Succeed()) }) By("Skip check", func() { - if m.VeleroCfg.DefaultClusterContext == "" || m.VeleroCfg.StandbyClusterContext == "" { + if m.VeleroCfg.ActiveClusterContext == "" || m.VeleroCfg.StandbyClusterContext == "" { Skip("Migration test needs 2 clusters") } - if m.useVolumeSnapshots && m.VeleroCfg.CloudProvider == test.Kind { + if m.UseVolumeSnapshots && m.VeleroCfg.CloudProvider == test.Kind { Skip(fmt.Sprintf("Volume snapshots not supported on %s", test.Kind)) } - if m.VeleroCfg.SnapshotMoveData && !m.useVolumeSnapshots { + if m.VeleroCfg.SnapshotMoveData && !m.UseVolumeSnapshots { Skip("FSB migration test is not needed in data mover scenario") } }) - m.kibishiiData = *kibishii.DefaultKibishiiData - m.kibishiiData.ExpectedNodes = 3 - m.CaseBaseName = "migration-" + m.UUIDgen - m.BackupName = m.CaseBaseName + "-backup" - m.RestoreName = m.CaseBaseName + "-restore" - m.NSIncluded = &[]string{m.CaseBaseName} - - m.RestoreArgs = []string{ - "create", "--namespace", m.VeleroCfg.VeleroNamespace, - "restore", m.RestoreName, - "--from-backup", m.BackupName, "--wait", - } - - // Message output by ginkgo - m.TestMsg = &framework.TestMSG{ - Desc: "Test migration workload on two clusters", - FailedMSG: "Fail to test migrate between two clusters", - Text: "Test back up on default cluster, restore on standby cluster", - } - - // Need to uninstall Velero on the default cluster. - if test.InstallVelero { - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) - defer ctxCancel() - Expect(veleroutil.VeleroUninstall(ctx, m.VeleroCfg)).To(Succeed()) - } - return nil } @@ -119,43 +93,43 @@ func (m *migrationE2E) Backup() error { OriginVeleroCfg := m.VeleroCfg var err error - if m.veleroCLI2Version.VeleroCLI == "" { + if m.VeleroCLI2Version.VeleroCLI == "" { //Assume tag of velero server image is identical to velero CLI version //Download velero CLI if it's empty according to velero CLI version By( fmt.Sprintf("Install the expected version Velero CLI %s", - m.veleroCLI2Version.VeleroVersion), + m.VeleroCLI2Version.VeleroVersion), func() { // "self" represents 1.14.x and future versions - if m.veleroCLI2Version.VeleroVersion == "self" { - m.veleroCLI2Version.VeleroCLI = m.VeleroCfg.VeleroCLI + if m.VeleroCLI2Version.VeleroVersion == "self" { + m.VeleroCLI2Version.VeleroCLI = m.VeleroCfg.VeleroCLI } else { OriginVeleroCfg, err = veleroutil.SetImagesToDefaultValues( OriginVeleroCfg, - m.veleroCLI2Version.VeleroVersion, + m.VeleroCLI2Version.VeleroVersion, ) Expect(err).To(Succeed(), "Fail to set images for the migrate-from Velero installation.") - m.veleroCLI2Version.VeleroCLI, err = veleroutil.InstallVeleroCLI( - m.veleroCLI2Version.VeleroVersion) + m.VeleroCLI2Version.VeleroCLI, err = veleroutil.InstallVeleroCLI( + m.VeleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) } }, ) } - By(fmt.Sprintf("Install Velero on default cluster (%s)", m.VeleroCfg.DefaultClusterContext), + By(fmt.Sprintf("Install Velero on default cluster (%s)", m.VeleroCfg.ActiveClusterContext), func() { Expect(k8sutil.KubectlConfigUseContext( - m.Ctx, m.VeleroCfg.DefaultClusterContext)).To(Succeed()) - OriginVeleroCfg.MigrateFromVeleroVersion = m.veleroCLI2Version.VeleroVersion - OriginVeleroCfg.VeleroCLI = m.veleroCLI2Version.VeleroCLI - OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient - OriginVeleroCfg.ClusterToInstallVelero = m.VeleroCfg.DefaultClusterName - OriginVeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.DefaultCLSServiceAccountName - OriginVeleroCfg.UseVolumeSnapshots = m.useVolumeSnapshots - OriginVeleroCfg.UseNodeAgent = !m.useVolumeSnapshots + m.Ctx, m.VeleroCfg.ActiveClusterContext)).To(Succeed()) + OriginVeleroCfg.MigrateFromVeleroVersion = m.VeleroCLI2Version.VeleroVersion + OriginVeleroCfg.VeleroCLI = m.VeleroCLI2Version.VeleroCLI + OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.ActiveClient + OriginVeleroCfg.ClusterToInstallVelero = m.VeleroCfg.ActiveClusterName + OriginVeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.ActiveCLSServiceAccountName + OriginVeleroCfg.UseVolumeSnapshots = m.UseVolumeSnapshots + OriginVeleroCfg.UseNodeAgent = !m.UseVolumeSnapshots version, err := veleroutil.GetVeleroVersion(m.Ctx, OriginVeleroCfg.VeleroCLI, true) Expect(err).To(Succeed(), "Fail to get Velero version") @@ -166,7 +140,7 @@ func (m *migrationE2E) Backup() error { } Expect(veleroutil.VeleroInstall(m.Ctx, &OriginVeleroCfg, false)).To(Succeed()) - if m.veleroCLI2Version.VeleroVersion != "self" { + if m.VeleroCLI2Version.VeleroVersion != "self" { Expect(veleroutil.CheckVeleroVersion( m.Ctx, OriginVeleroCfg.VeleroCLI, @@ -179,7 +153,7 @@ func (m *migrationE2E) Backup() error { By("Create namespace for sample workload", func() { Expect(k8sutil.CreateNamespace( m.Ctx, - *m.VeleroCfg.DefaultClient, + *m.VeleroCfg.ActiveClient, m.CaseBaseName, )).To(Succeed(), fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", @@ -189,14 +163,14 @@ func (m *migrationE2E) Backup() error { By("Deploy sample workload of Kibishii", func() { Expect(kibishii.KibishiiPrepareBeforeBackup( m.Ctx, - *OriginVeleroCfg.DefaultClient, + *OriginVeleroCfg.ActiveClient, OriginVeleroCfg.CloudProvider, m.CaseBaseName, OriginVeleroCfg.RegistryCredentialFile, OriginVeleroCfg.Features, OriginVeleroCfg.KibishiiDirectory, OriginVeleroCfg.UseVolumeSnapshots, - &m.kibishiiData, + &m.KibishiiData, )).To(Succeed()) }) @@ -204,11 +178,10 @@ func (m *migrationE2E) Backup() error { m.BackupArgs = []string{ "create", "--namespace", m.VeleroCfg.VeleroNamespace, "backup", m.BackupName, - "--include-namespaces", strings.Join(*m.NSIncluded, ","), "--wait", } - if m.useVolumeSnapshots { + if m.UseVolumeSnapshots { m.BackupArgs = append(m.BackupArgs, "--snapshot-volumes=true") } else { m.BackupArgs = append(m.BackupArgs, "--default-volumes-to-fs-backup") @@ -236,7 +209,7 @@ func (m *migrationE2E) Backup() error { }) }) - if m.useVolumeSnapshots { + if m.UseVolumeSnapshots { // Only wait for the snapshots.backupdriver.cnsdp.vmware.com // when the vSphere plugin is used. if OriginVeleroCfg.HasVspherePlugin { @@ -246,7 +219,7 @@ func (m *migrationE2E) Backup() error { context.Background(), time.Hour, m.CaseBaseName, - m.kibishiiData.ExpectedNodes, + m.KibishiiData.ExpectedNodes, ), ).To(Succeed()) }) @@ -258,7 +231,7 @@ func (m *migrationE2E) Backup() error { if OriginVeleroCfg.SnapshotMoveData { //VolumeSnapshotContent should be deleted after data movement _, err := util.CheckVolumeSnapshotCR( - *m.VeleroCfg.DefaultClient, + *m.VeleroCfg.ActiveClient, map[string]string{"namespace": m.CaseBaseName}, 0, ) @@ -278,12 +251,12 @@ func (m *migrationE2E) Backup() error { By("Snapshot should be created in cloud object store with retain policy", func() { snapshotCheckPoint, err = veleroutil.GetSnapshotCheckPoint( - *OriginVeleroCfg.DefaultClient, + *OriginVeleroCfg.ActiveClient, OriginVeleroCfg, - m.kibishiiData.ExpectedNodes, + m.KibishiiData.ExpectedNodes, m.CaseBaseName, m.BackupName, - kibishii.GetKibishiiPVCNameList(m.kibishiiData.ExpectedNodes), + kibishii.GetKibishiiPVCNameList(m.KibishiiData.ExpectedNodes), ) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") @@ -335,7 +308,7 @@ func (m *migrationE2E) Restore() error { StandbyVeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient StandbyVeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName StandbyVeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.StandbyCLSServiceAccountName - StandbyVeleroCfg.UseNodeAgent = !m.useVolumeSnapshots + StandbyVeleroCfg.UseNodeAgent = !m.UseVolumeSnapshots if StandbyVeleroCfg.SnapshotMoveData { StandbyVeleroCfg.UseNodeAgent = true // For SnapshotMoveData pipelines, we should use standby cluster setting @@ -343,7 +316,7 @@ func (m *migrationE2E) Restore() error { // In nightly CI, StandbyClusterPlugins is set properly // if pipeline is for SnapshotMoveData. StandbyVeleroCfg.Plugins = m.VeleroCfg.StandbyClusterPlugins - StandbyVeleroCfg.ObjectStoreProvider = m.VeleroCfg.StandbyClusterObjectStoreProvider + StandbyVeleroCfg.ObjectStoreProvider = m.VeleroCfg.StandbyObjectStoreProvider } Expect(veleroutil.VeleroInstall( @@ -401,71 +374,10 @@ func (m *migrationE2E) Verify() error { *m.VeleroCfg.StandbyClient, m.CaseBaseName, m.Ctx, - &m.kibishiiData, + &m.KibishiiData, "", )).To(Succeed(), "Fail to verify workload after restore") }) return nil } - -func (m *migrationE2E) Clean() error { - By("Clean resource on default cluster.", func() { - Expect(m.TestCase.Clean()).To(Succeed()) - }) - - By("Clean resource on standby cluster.", func() { - Expect(k8sutil.KubectlConfigUseContext( - m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed()) - m.VeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient - m.VeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName - - By("Delete StorageClasses created by E2E") - Expect( - k8sutil.DeleteStorageClass( - m.Ctx, - *m.VeleroCfg.ClientToInstallVelero, - test.StorageClassName, - ), - ).To(Succeed()) - Expect( - k8sutil.DeleteStorageClass( - m.Ctx, - *m.VeleroCfg.ClientToInstallVelero, - test.StorageClassName2, - ), - ).To(Succeed()) - - if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) && - m.VeleroCfg.UseVolumeSnapshots { - By("Delete VolumeSnapshotClass created by E2E") - Expect( - k8sutil.KubectlDeleteByFile( - m.Ctx, - fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", - m.VeleroCfg.StandbyClusterCloudProvider), - ), - ).To(Succeed()) - } - - Expect(veleroutil.VeleroUninstall(m.Ctx, m.VeleroCfg)).To(Succeed()) - - Expect( - k8sutil.DeleteNamespace( - m.Ctx, - *m.VeleroCfg.StandbyClient, - m.CaseBaseName, - true, - ), - ).To(Succeed()) - }) - - By("Switch to default KubeConfig context", func() { - Expect(k8sutil.KubectlConfigUseContext( - m.Ctx, - m.VeleroCfg.DefaultClusterContext, - )).To(Succeed()) - }) - - return nil -} diff --git a/test/e2e/parallelfilesdownload/parallel_files_download.go b/test/e2e/parallelfilesdownload/parallel_files_download.go index a678c6c80e..1cdfe5c447 100644 --- a/test/e2e/parallelfilesdownload/parallel_files_download.go +++ b/test/e2e/parallelfilesdownload/parallel_files_download.go @@ -23,12 +23,12 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) type ParallelFilesDownload struct { - TestCase + BRCase parallel string namespace string pod string @@ -44,7 +44,11 @@ var ParallelFilesDownloadTest func() = TestFunc(&ParallelFilesDownload{}) func (p *ParallelFilesDownload) Init() error { // generate random number as UUIDgen and set one default timeout duration - p.TestCase.Init() + p.BRCase.Init() + + if p.VeleroCfg.UploaderType == UploaderTypeRestic { + Skip("Skip Parallel Files cases for environments using Restic as uploader.") + } // generate variable names based on CaseBaseName + UUIDgen p.CaseBaseName = "parallel-files-download" + p.UUIDgen diff --git a/test/e2e/parallelfilesupload/parallel_files_upload.go b/test/e2e/parallelfilesupload/parallel_files_upload.go index 5478e35186..681d41efaf 100644 --- a/test/e2e/parallelfilesupload/parallel_files_upload.go +++ b/test/e2e/parallelfilesupload/parallel_files_upload.go @@ -23,12 +23,12 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) type ParallelFilesUpload struct { - TestCase + BRCase parallel string namespace string pod string @@ -43,7 +43,11 @@ var ParallelFilesUploadTest func() = TestFunc(&ParallelFilesUpload{}) func (p *ParallelFilesUpload) Init() error { // generate random number as UUIDgen and set one default timeout duration - p.TestCase.Init() + p.BRCase.Init() + + if p.VeleroCfg.UploaderType == UploaderTypeRestic { + Skip("Skip Parallel Files cases for environments using Restic as uploader.") + } // generate variable names based on CaseBaseName + UUIDgen p.CaseBaseName = "parallel-files-upload" + p.UUIDgen diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index b58cb8a55b..b47c682d14 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -10,13 +10,13 @@ import ( "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/common" . "github.com/vmware-tanzu/velero/test/util/k8s" ) type PVBackupFiltering struct { - TestCase + BRCase annotation string podsList [][]string volumesList [][]string @@ -31,7 +31,7 @@ var OptInPVBackupTest func() = TestFunc(&PVBackupFiltering{annotation: OPT_IN_AN var OptOutPVBackupTest func() = TestFunc(&PVBackupFiltering{annotation: OPT_OUT_ANN, id: "opt-out"}) func (p *PVBackupFiltering) Init() error { - p.TestCase.Init() + p.BRCase.Init() p.CaseBaseName = "pv-filter-" + p.UUIDgen p.BackupName = "backup-" + p.CaseBaseName + p.id p.RestoreName = "restore-" + p.CaseBaseName + p.id diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index 8cfce438d5..89ee513887 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -23,12 +23,12 @@ import ( "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) type FilteringCase struct { - TestCase + BRCase IsTestInBackup bool replica int32 labels map[string]string @@ -39,7 +39,7 @@ var testInBackup = FilteringCase{IsTestInBackup: true} var testInRestore = FilteringCase{IsTestInBackup: false} func (f *FilteringCase) Init() error { - f.TestCase.Init() + f.BRCase.Init() f.replica = int32(2) f.labels = map[string]string{"resourcefiltering": "true"} diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index b90d63dd87..fdff019050 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -25,7 +25,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) diff --git a/test/e2e/resource-filtering/exclude_namespaces.go b/test/e2e/resource-filtering/exclude_namespaces.go index 1b8e5da550..1cc2827ef7 100644 --- a/test/e2e/resource-filtering/exclude_namespaces.go +++ b/test/e2e/resource-filtering/exclude_namespaces.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) diff --git a/test/e2e/resource-filtering/exclude_resources.go b/test/e2e/resource-filtering/exclude_resources.go index 5c2f37e737..1cd245742a 100644 --- a/test/e2e/resource-filtering/exclude_resources.go +++ b/test/e2e/resource-filtering/exclude_resources.go @@ -25,7 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) diff --git a/test/e2e/resource-filtering/include_namespaces.go b/test/e2e/resource-filtering/include_namespaces.go index d511de2123..a618fb5c1a 100644 --- a/test/e2e/resource-filtering/include_namespaces.go +++ b/test/e2e/resource-filtering/include_namespaces.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) diff --git a/test/e2e/resource-filtering/include_resources.go b/test/e2e/resource-filtering/include_resources.go index 23199d2de2..dbc70b04a2 100644 --- a/test/e2e/resource-filtering/include_resources.go +++ b/test/e2e/resource-filtering/include_resources.go @@ -24,7 +24,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index 1d566b5853..7616b6f071 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -24,7 +24,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) diff --git a/test/e2e/resourcemodifiers/resource_modifiers.go b/test/e2e/resourcemodifiers/resource_modifiers.go index e3bd9ea11d..ea19a066fc 100644 --- a/test/e2e/resourcemodifiers/resource_modifiers.go +++ b/test/e2e/resourcemodifiers/resource_modifiers.go @@ -24,7 +24,7 @@ import ( . "github.com/onsi/gomega" "github.com/pkg/errors" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -44,7 +44,7 @@ resourceModifierRules: ` type ResourceModifiersCase struct { - TestCase + BRCase cmName, yamlConfig string } @@ -52,7 +52,7 @@ var ResourceModifiersTest func() = TestFunc(&ResourceModifiersCase{}) func (r *ResourceModifiersCase) Init() error { // generate random number as UUIDgen and set one default timeout duration - r.TestCase.Init() + r.BRCase.Init() // generate variable names based on CaseBaseName + UUIDgen r.CaseBaseName = "resource-modifiers-" + r.UUIDgen @@ -138,7 +138,7 @@ func (r *ResourceModifiersCase) Clean() error { return err } - return r.GetTestCase().Clean() // only clean up resources in test namespace + return r.BRCase.Clean() // only clean up resources in test namespace } return nil diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index 94e238eda3..8fe4026483 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -47,7 +47,7 @@ volumePolicies: ` type ResourcePoliciesCase struct { - TestCase + BRCase cmName, yamlConfig string } @@ -55,7 +55,7 @@ var ResourcePoliciesTest func() = TestFunc(&ResourcePoliciesCase{}) func (r *ResourcePoliciesCase) Init() error { // generate random number as UUIDgen and set one default timeout duration - r.TestCase.Init() + r.BRCase.Init() // generate variable names based on CaseBaseName + UUIDgen r.CaseBaseName = "resource-policies-" + r.UUIDgen @@ -180,7 +180,7 @@ func (r *ResourcePoliciesCase) Clean() error { return err } - return r.GetTestCase().Clean() // only clean up resources in test namespace + return r.BRCase.Clean() // only clean up resources in test namespace } return nil diff --git a/test/e2e/scale/multiple_namespaces.go b/test/e2e/scale/multiple_namespaces.go index d63880c6b1..59985db160 100644 --- a/test/e2e/scale/multiple_namespaces.go +++ b/test/e2e/scale/multiple_namespaces.go @@ -18,7 +18,7 @@ package scale import ( basic "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check" - . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/framework" ) var MultiNSBackupRestore func() = TestFunc(&basic.MultiNSBackup{IsScalTest: true}) diff --git a/test/e2e/schedule/in_progress.go b/test/e2e/schedule/in_progress.go index 3a148a8c1f..71197b9f7a 100644 --- a/test/e2e/schedule/in_progress.go +++ b/test/e2e/schedule/in_progress.go @@ -13,7 +13,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/test" - framework "github.com/vmware-tanzu/velero/test/e2e/test" + framework "github.com/vmware-tanzu/velero/test/e2e/framework" k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" veleroutil "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -21,7 +21,7 @@ import ( var ScheduleInProgressTest func() = framework.TestFunc(&InProgressCase{}) type InProgressCase struct { - framework.TestCase + framework.BRCase namespace string ScheduleName string ScheduleArgs []string @@ -33,7 +33,7 @@ type InProgressCase struct { } func (s *InProgressCase) Init() error { - Expect(s.TestCase.Init()).To(Succeed()) + Expect(s.BRCase.Init()).To(Succeed()) s.CaseBaseName = "schedule-backup-creation-test" + s.UUIDgen s.ScheduleName = "schedule-" + s.CaseBaseName @@ -189,7 +189,7 @@ func (s *InProgressCase) Clean() error { s.ScheduleName, ), ).To(Succeed()) - Expect(s.TestCase.Clean()).To(Succeed()) + Expect(s.BRCase.Clean()).To(Succeed()) } return nil diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index 238a014c75..cad74c82d5 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -31,7 +31,7 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - framework "github.com/vmware-tanzu/velero/test/e2e/test" + framework "github.com/vmware-tanzu/velero/test/e2e/framework" k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" veleroutil "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -43,11 +43,11 @@ type OrderedResources struct { ScheduleName string OrderResource map[string]string ScheduleArgs []string - framework.TestCase + framework.BRCase } func (o *OrderedResources) Init() error { - Expect(o.TestCase.Init()).To(Succeed()) + Expect(o.BRCase.Init()).To(Succeed()) o.CaseBaseName = "ordered-resources-" + o.UUIDgen o.ScheduleName = "schedule-" + o.CaseBaseName @@ -224,7 +224,7 @@ func (o *OrderedResources) Clean() error { o.ScheduleName, )).To(Succeed()) - Expect(o.TestCase.Clean()).To(Succeed()) + Expect(o.BRCase.Clean()).To(Succeed()) } return nil diff --git a/test/e2e/schedule/periodical.go b/test/e2e/schedule/periodical.go index 330356e998..2ba5c3c487 100644 --- a/test/e2e/schedule/periodical.go +++ b/test/e2e/schedule/periodical.go @@ -11,13 +11,13 @@ import ( "k8s.io/apimachinery/pkg/util/wait" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - framework "github.com/vmware-tanzu/velero/test/e2e/test" + framework "github.com/vmware-tanzu/velero/test/e2e/framework" k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" veleroutil "github.com/vmware-tanzu/velero/test/util/velero" ) type PeriodicalCase struct { - framework.TestCase + framework.BRCase ScheduleName string ScheduleArgs []string Period int // The minimum unit is minute. @@ -26,7 +26,7 @@ type PeriodicalCase struct { var SchedulePeriodicalTest func() = framework.TestFunc(&PeriodicalCase{}) func (n *PeriodicalCase) Init() error { - Expect(n.TestCase.Init()).To(Succeed()) + Expect(n.BRCase.Init()).To(Succeed()) n.CaseBaseName = "schedule-backup-" + n.UUIDgen n.NSIncluded = &[]string{n.CaseBaseName} @@ -203,7 +203,7 @@ func (n *PeriodicalCase) Clean() error { ), ).To(Succeed()) - Expect(n.TestCase.Clean()).To(Succeed()) + Expect(n.BRCase.Clean()).To(Succeed()) } return nil diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go deleted file mode 100644 index 78704a4bba..0000000000 --- a/test/e2e/test/test.go +++ /dev/null @@ -1,256 +0,0 @@ -/* -Copyright the Velero contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package test - -import ( - "context" - "fmt" - "math/rand/v2" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/pkg/errors" - - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - . "github.com/vmware-tanzu/velero/test" - . "github.com/vmware-tanzu/velero/test/util/k8s" - . "github.com/vmware-tanzu/velero/test/util/velero" -) - -/* -The VeleroBackupRestoreTest interface is just could be suit for the cases that follow the test flow of -create resources, backup, delete test resource, restore and verify. -And the cases have similar execute function and similar data. it's both fine for you to use it or not which -depends on your test patterns. -*/ -type VeleroBackupRestoreTest interface { - Init() error - CreateResources() error - Backup() error - Destroy() error - Restore() error - Verify() error - Clean() error - Start() error - GetTestMsg() *TestMSG - GetTestCase() *TestCase -} - -type TestMSG struct { - Desc string - Text string - FailedMSG string -} - -type TestCase struct { - BackupName string - RestoreName string - CaseBaseName string - BackupArgs []string - RestoreArgs []string - NamespacesTotal int - TestMsg *TestMSG - Client TestClient - NSIncluded *[]string - UseVolumeSnapshots bool - VeleroCfg VeleroConfig - RestorePhaseExpect velerov1api.RestorePhase - Ctx context.Context - CtxCancel context.CancelFunc - UUIDgen string -} - -func TestFunc(test VeleroBackupRestoreTest) func() { - return func() { - TestIt(test) - } -} - -func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { - return func() { - for k := range tests { - TestIt(tests[k]) - } - } -} - -func TestIt(test VeleroBackupRestoreTest) error { - It("Run E2E test case", func() { - Expect(test.Init()).To(Succeed()) - - Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) - }) - return nil -} -func (t *TestCase) Init() error { - t.UUIDgen = t.GenerateUUID() - t.VeleroCfg = VeleroCfg - t.Client = *t.VeleroCfg.ClientToInstallVelero - return nil -} - -func (t *TestCase) GenerateUUID() string { - return fmt.Sprintf("%08d", rand.IntN(100000000)) -} - -func (t *TestCase) CreateResources() error { - return nil -} - -func (t *TestCase) Backup() error { - veleroCfg := t.GetTestCase().VeleroCfg - - By("Start to backup ......", func() { - Expect(VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs)).To(Succeed(), func() string { - RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") - return "Failed to backup resources" - }) - }) - - return nil -} - -func (t *TestCase) Destroy() error { - By(fmt.Sprintf("Start to destroy namespace %s......", t.CaseBaseName), func() { - Expect(CleanupNamespacesWithPoll(t.Ctx, t.Client, t.CaseBaseName)).To(Succeed(), "Could cleanup retrieve namespaces") - }) - return nil -} - -func (t *TestCase) Restore() error { - if len(t.RestoreArgs) == 0 { - return nil - } - - veleroCfg := t.GetTestCase().VeleroCfg - - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - By("Waiting 5 minutes to make sure the snapshots are ready...", func() { - if t.UseVolumeSnapshots && veleroCfg.CloudProvider != Vsphere { - time.Sleep(5 * time.Minute) - } - }) - - By("Start to restore ......", func() { - if t.RestorePhaseExpect == "" { - t.RestorePhaseExpect = velerov1api.RestorePhaseCompleted - } - Expect(VeleroRestoreExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs, t.RestorePhaseExpect)).To(Succeed(), func() string { - RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", t.RestoreName) - return "Fail to restore workload" - }) - }) - return nil -} - -func (t *TestCase) Verify() error { - return nil -} - -func (t *TestCase) Start() error { - t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) - veleroCfg := t.GetTestCase().VeleroCfg - - if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == AWS) && - strings.Contains(t.GetTestCase().CaseBaseName, "nodeport") { - Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") - } - - if veleroCfg.UploaderType == UploaderTypeRestic && - strings.Contains(t.GetTestCase().CaseBaseName, "ParallelFiles") { - Skip("Skip Parallel Files upload and download test cases for environments using Restic as uploader.") - } - return nil -} - -func (t *TestCase) Clean() error { - veleroCfg := t.GetTestCase().VeleroCfg - if CurrentSpecReport().Failed() && veleroCfg.FailFast { - fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") - } else { - By(fmt.Sprintf("Clean namespace with prefix %s after test", t.CaseBaseName), func() { - if err := CleanupNamespaces(t.Ctx, t.Client, t.CaseBaseName); err != nil { - fmt.Println("Fail to cleanup namespaces: ", err) - } - }) - By("Clean backups after test", func() { - veleroCfg.ClientToInstallVelero = &t.Client - if err := DeleteAllBackups(t.Ctx, &veleroCfg); err != nil { - fmt.Println("Fail to clean backups after test: ", err) - } - }) - } - - return nil -} - -func (t *TestCase) GetTestMsg() *TestMSG { - return t.TestMsg -} - -func (t *TestCase) GetTestCase() *TestCase { - return t -} - -func RunTestCase(test VeleroBackupRestoreTest) error { - if test == nil { - return errors.New("No case should be tested") - } - fmt.Println("Running case: ", test.GetTestMsg().Text) - test.Start() - defer test.GetTestCase().CtxCancel() - - fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) - - if InstallVelero { - Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, test.GetTestCase().VeleroCfg)).To(Succeed()) - } - - defer test.Clean() - - fmt.Printf("CreateResources %s\n", time.Now().Format("2006-01-02 15:04:05")) - err := test.CreateResources() - if err != nil { - return err - } - fmt.Printf("Backup %s\n", time.Now().Format("2006-01-02 15:04:05")) - err = test.Backup() - if err != nil { - return err - } - fmt.Printf("Destroy %s\n", time.Now().Format("2006-01-02 15:04:05")) - err = test.Destroy() - if err != nil { - return err - } - fmt.Printf("Restore %s\n", time.Now().Format("2006-01-02 15:04:05")) - err = test.Restore() - if err != nil { - return err - } - fmt.Printf("Verify %s\n", time.Now().Format("2006-01-02 15:04:05")) - err = test.Verify() - if err != nil { - return err - } - fmt.Printf("Finish run test %s\n", time.Now().Format("2006-01-02 15:04:05")) - return nil -} diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index cb4522d699..e4229583d8 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -68,7 +68,7 @@ func init() { flag.DurationVar(&VeleroCfg.PodVolumeOperationTimeout, "pod-volume-operation-timeout", 360*time.Minute, "Timeout for pod volume operations. Optional.") //vmware-tanzu-experiments flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.") - flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "Default cluster context for migration test.") + flag.StringVar(&VeleroCfg.ActiveClusterContext, "default-cluster-context", "", "Default cluster context for migration test.") flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "kopia", "Identify persistent volume backup uploader.") flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "Identify persistent volume backup uploader.") flag.StringVar(&VeleroCfg.NFSServerPath, "nfs-server-path", "", "the path of nfs server") @@ -84,7 +84,7 @@ func initConfig() error { if err != nil { return errors.WithStack(err) } - VeleroCfg.DefaultClient = &cli + VeleroCfg.ActiveClient = &cli ReportData = &E2EReport{ TestDescription: VeleroCfg.TestCaseDescribe, diff --git a/test/perf/test/test.go b/test/perf/test/test.go index 1bc8a3fc06..778014d68d 100644 --- a/test/perf/test/test.go +++ b/test/perf/test/test.go @@ -98,7 +98,7 @@ func (t *TestCase) Init() error { t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 6*time.Hour) t.NSExcluded = &[]string{"kube-system", "velero", "default", "kube-public", "kube-node-lease"} t.UUIDgen = t.GenerateUUID() - t.Client = *VeleroCfg.DefaultClient + t.Client = *VeleroCfg.ActiveClient t.timer = &metrics.TimeMetrics{ Name: "Total time cost", TimeInfo: map[string]metrics.TimeSpan{"Total time cost": { @@ -281,7 +281,7 @@ func (t *TestCase) MonitorMetircs(ctx context.Context, collectors *metrics.Metri for _, pod := range veleroPodList { podMetrics := &metrics.PodMetrics{ Ctx: ctx, - Client: VeleroCfg.DefaultClient.MetricsClient, + Client: VeleroCfg.ActiveClient.MetricsClient, Metrics: make(map[string]int64), PodName: pod, Namespace: VeleroCfg.VeleroNamespace, diff --git a/test/types.go b/test/types.go index 35a1502c2d..d1891fa4a9 100644 --- a/test/types.go +++ b/test/types.go @@ -72,58 +72,58 @@ var ReportData *E2EReport type VeleroConfig struct { VeleroCfgInPerf install.Options - VeleroCLI string - VeleroImage string - VeleroVersion string - CloudCredentialsFile string - BSLConfig string - BSLBucket string - BSLPrefix string - VSLConfig string - CloudProvider string - ObjectStoreProvider string - VeleroNamespace string - AdditionalBSLProvider string - AdditionalBSLBucket string - AdditionalBSLPrefix string - AdditionalBSLConfig string - AdditionalBSLCredentials string - RegistryCredentialFile string - RestoreHelperImage string - UpgradeFromVeleroVersion string - UpgradeFromVeleroCLI string - MigrateFromVeleroVersion string - MigrateFromVeleroCLI string - Plugins string - AddBSLPlugins string - KibishiiDirectory string - GCFrequency string - DefaultClusterContext string - StandbyClusterContext string - ClientToInstallVelero *k8s.TestClient - DefaultClient *k8s.TestClient - StandbyClient *k8s.TestClient - ClusterToInstallVelero string - DefaultClusterName string - StandbyClusterName string - ProvideSnapshotsVolumeParam bool - VeleroServerDebugMode bool - SnapshotMoveData bool - DataMoverPlugin string - StandbyClusterCloudProvider string - StandbyClusterPlugins string - StandbyClusterObjectStoreProvider string - DebugVeleroPodRestart bool - IsUpgradeTest bool - WithoutDisableInformerCacheParam bool - DisableInformerCache bool - CreateClusterRoleBinding bool - DefaultCLSServiceAccountName string - StandbyCLSServiceAccountName string - ServiceAccountNameToInstall string - EKSPolicyARN string - FailFast bool - HasVspherePlugin bool + VeleroCLI string + VeleroImage string + VeleroVersion string + CloudCredentialsFile string + BSLConfig string + BSLBucket string + BSLPrefix string + VSLConfig string + CloudProvider string + ObjectStoreProvider string + VeleroNamespace string + AdditionalBSLProvider string + AdditionalBSLBucket string + AdditionalBSLPrefix string + AdditionalBSLConfig string + AdditionalBSLCredentials string + RegistryCredentialFile string + RestoreHelperImage string + UpgradeFromVeleroVersion string + UpgradeFromVeleroCLI string + MigrateFromVeleroVersion string + MigrateFromVeleroCLI string + Plugins string + AddBSLPlugins string + KibishiiDirectory string + GCFrequency string + ActiveClusterContext string + StandbyClusterContext string + ClientToInstallVelero *k8s.TestClient + ActiveClient *k8s.TestClient + StandbyClient *k8s.TestClient + ClusterToInstallVelero string + ActiveClusterName string + StandbyClusterName string + ProvideSnapshotsVolumeParam bool + VeleroServerDebugMode bool + SnapshotMoveData bool + DataMoverPlugin string + StandbyClusterCloudProvider string + StandbyClusterPlugins string + StandbyObjectStoreProvider string + DebugVeleroPodRestart bool + IsUpgradeTest bool + WithoutDisableInformerCacheParam bool + DisableInformerCache bool + CreateClusterRoleBinding bool + ActiveCLSServiceAccountName string + StandbyCLSServiceAccountName string + ServiceAccountNameToInstall string + EKSPolicyARN string + FailFast bool + HasVspherePlugin bool } type VeleroCfgInPerf struct { diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 027ca43048..c48d1e99db 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -27,7 +27,6 @@ import ( "time" "github.com/pkg/errors" - "golang.org/x/exp/slices" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -53,9 +52,17 @@ type installOptions struct { WithoutDisableInformerCacheParam bool } -func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyCluster bool) error { +func VeleroInstall( + ctx context.Context, + veleroCfg *test.VeleroConfig, + isStandbyCluster bool, +) error { fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05")) + cloudProvider := veleroCfg.CloudProvider + objectStoreProvider := veleroCfg.ObjectStoreProvider + client := veleroCfg.ActiveClient + // veleroCfg struct including a set of BSL params and a set of additional BSL params, // additional BSL set is for additional BSL test only, so only default BSL set is effective // for VeleroInstall(). @@ -69,19 +76,9 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC // the only one veleroCfg setting is provided as current design, since it will not introduce any issues as // we can predict, so keep it intact for now. if isStandbyCluster { - veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider - } - - if slices.Contains(test.PublicCloudProviders, veleroCfg.CloudProvider) { - fmt.Println("For public cloud platforms, object store plugin provider will be set as cloud provider") - // If ObjectStoreProvider is not provided, then using the value same as CloudProvider - if veleroCfg.ObjectStoreProvider == "" { - veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider - } - } else { - if veleroCfg.ObjectStoreProvider == "" { - return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Must have an object store provider - } + cloudProvider = veleroCfg.StandbyClusterCloudProvider + objectStoreProvider = veleroCfg.StandbyObjectStoreProvider + client = veleroCfg.StandbyClient } pluginsTmp, err := getPlugins(ctx, *veleroCfg) @@ -94,23 +91,17 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC } // TODO - handle this better - if veleroCfg.CloudProvider == test.Vsphere { - // We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the - // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume - // Snapshot location specified - if veleroCfg.ObjectStoreProvider == "" { - veleroCfg.ObjectStoreProvider = test.AWS - } - + if cloudProvider == test.Vsphere { if err := cleanVSpherePluginConfig( - veleroCfg.ClientToInstallVelero.ClientGo, + client.ClientGo, veleroCfg.VeleroNamespace, test.VeleroVSphereSecretName, test.VeleroVSphereConfigMapName, ); err != nil { return errors.WithMessagef(err, "Failed to clear up vsphere plugin config %s namespace", veleroCfg.VeleroNamespace) } - if err := generateVSpherePlugin(veleroCfg); err != nil { + + if err := generateVSpherePlugin(client, veleroCfg); err != nil { return errors.WithMessagef(err, "Failed to config vsphere plugin") } } @@ -173,12 +164,11 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC // generateVSpherePlugin refers to // https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md -func generateVSpherePlugin(veleroCfg *test.VeleroConfig) error { - cli := veleroCfg.ClientToInstallVelero +func generateVSpherePlugin(client *k8s.TestClient, veleroCfg *test.VeleroConfig) error { if err := k8s.CreateNamespace( context.Background(), - *cli, + *client, veleroCfg.VeleroNamespace, ); err != nil { return errors.WithMessagef( @@ -190,7 +180,7 @@ func generateVSpherePlugin(veleroCfg *test.VeleroConfig) error { clusterFlavor := "VANILLA" - if err := createVCCredentialSecret(cli.ClientGo, veleroCfg.VeleroNamespace); err != nil { + if err := createVCCredentialSecret(client.ClientGo, veleroCfg.VeleroNamespace); err != nil { // For TKGs/uTKG the VC secret is not supposed to exist. if apierrors.IsNotFound(err) { clusterFlavor = "GUEST" @@ -204,7 +194,7 @@ func generateVSpherePlugin(veleroCfg *test.VeleroConfig) error { } _, err := k8s.CreateConfigMap( - cli.ClientGo, + client.ClientGo, veleroCfg.VeleroNamespace, test.VeleroVSphereConfigMapName, nil, @@ -223,7 +213,7 @@ func generateVSpherePlugin(veleroCfg *test.VeleroConfig) error { } if err := k8s.WaitForConfigMapComplete( - cli.ClientGo, + client.ClientGo, veleroCfg.VeleroNamespace, test.VeleroVSphereConfigMapName, ); err != nil { diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index c57ceae4c4..31ff19cc05 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -46,9 +46,11 @@ import ( cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test" common "github.com/vmware-tanzu/velero/test/util/common" util "github.com/vmware-tanzu/velero/test/util/csi" + "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -245,10 +247,16 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, io.BucketName = veleroCfg.BSLBucket io.Prefix = veleroCfg.BSLPrefix io.BackupStorageConfig = flag.NewMap() - io.BackupStorageConfig.Set(veleroCfg.BSLConfig) + if err := io.BackupStorageConfig.Set(veleroCfg.BSLConfig); err != nil { + fmt.Printf("Fail to set the BSL %s.\n", err.Error()) + return nil, err + } io.VolumeSnapshotConfig = flag.NewMap() - io.VolumeSnapshotConfig.Set(veleroCfg.VSLConfig) + if err := io.VolumeSnapshotConfig.Set(veleroCfg.VSLConfig); err != nil { + fmt.Printf("Fail to set the VSL %s.\n", err.Error()) + return nil, err + } io.Plugins = flag.NewStringArray(plugins...) io.Features = veleroCfg.Features @@ -1818,3 +1826,52 @@ func KubectlGetAllDeleteBackupRequest(ctx context.Context, backupName, veleroNam return common.GetListByCmdPipes(ctx, cmds) } + +func DeleteVeleroAndRelatedResources( + ctx context.Context, + k8sContext string, + client *k8s.TestClient, + provider string, +) error { + if err := k8s.KubectlConfigUseContext(ctx, k8sContext); err != nil { + fmt.Printf("Fail to switch to k8s context %s: %s.\n", k8sContext, err.Error()) + return err + } + + if err := k8s.DeleteStorageClass( + ctx, + *client, + test.StorageClassName, + ); err != nil { + fmt.Printf("Fail to delete StorageClass %s: %s.\n", test.StorageClassName, err.Error()) + return err + } + + if err := k8s.DeleteStorageClass( + ctx, + *client, + test.StorageClassName2, + ); err != nil { + fmt.Printf("Fail to delete StorageClass %s: %s.\n", test.StorageClassName2, err.Error()) + return err + } + + if strings.EqualFold(test.VeleroCfg.Features, test.FeatureCSI) && + test.VeleroCfg.UseVolumeSnapshots { + if err := k8s.KubectlDeleteByFile( + ctx, + fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", + provider), + ); err != nil { + fmt.Printf("Fail to delete VolumeSnapshotClass: %s.\n", err.Error()) + return err + } + } + + if err := (VeleroUninstall(ctx, test.VeleroCfg)); err != nil { + fmt.Printf("Fail to uninstall Velero: %s.\n", err.Error()) + return err + } + + return nil +}