diff --git a/docs/guest.md b/docs/guest.md index 9d9737d1..3c164815 100644 --- a/docs/guest.md +++ b/docs/guest.md @@ -16,6 +16,8 @@ This document discusses the velero vSphere plugin installation process in a **Ta | Velero Plugin for vSphere Version | vSphere Version | Kubernetes Version | vSphere CSI Driver Version | Velero Version | vSphere Plugin Deprecated | vSphere Plugin EOL Date | |-----------------------------------|--------------------|--------------------|----------------------------|----------------|------------|---------------| +| 1.5.1 | 8.0U1 | 1.24 | Bundled with TKGS | 1.10.2 | No | N/A | +| 1.5.1 | 8.0c | 1.24 | Bundled with TKGS | 1.10.2 | No | N/A | | 1.4.2 | 8.0 | 1.20-1.23 | Bundled with TKGS | 1.9.2 | No | N/A | | 1.4.2 | 7.0U3h | 1.22 | Bundled with TKGS | 1.9.2 | No | N/A | | 1.4.0 | 7.0U1c/P02 - 7.0U3 | 1.19-1.22 | Bundled with TKGS | 1.8.1 | No | N/A | diff --git a/docs/supervisor-datamgr.md b/docs/supervisor-datamgr.md index 93d0a27b..556c7abd 100644 --- a/docs/supervisor-datamgr.md +++ b/docs/supervisor-datamgr.md @@ -6,9 +6,8 @@ In a vSphere with Tanzu environment, the Data Manager should be installed as a V This document discusses the installation procedure for backup data manager for **Velero plugin for vSphere** with kubernetes. -## Changes with release 1.1.0 +## Best Practice -Support of vSphere with Tanzu (Supervisor cluster and TKGS cluster) is being added with the release 1.1.0 release. - As a best practice, Data Manager VMs should be installed on the vSphere compute cluster where the workload cluster is installed. - Each Data Manager VM can serve upload/download tasks from a single workload cluster and the TKGS clusters in it. @@ -18,7 +17,7 @@ Support of vSphere with Tanzu (Supervisor cluster and TKGS cluster) is being add It is recommended that the Kubernetes backup/restore traffic be separated from the vSphere management network on a workload cluster. A backup network can be configured as an NSX-T network or traditional TDS network. We can add a VMkernel NIC on each ESXi host in the cluster and set the ```vSphereBackupNFC``` on that NIC. This enables backup network traffic to be sent through that NIC. If the ```vSphereBackupNFC``` is not enabled on the VMkernel NIC, the backup traffic will be sent on the management network. -More details can be found in the [vSphere documentation](https://code.vmware.com/docs/12628/virtual-disk-development-kit-programming-guide/GUID-5D166ED1-7205-4110-8D72-0C51BB63CC3D.html). +More details can be found in the [vSphere documentation](https://docs.vmware.com/en/VMware-vSphere/8.0/vsphere-networking/GUID-7BC73116-C4A6-411D-8A32-AD5B7A3D5493.html). Here are some ways to setup a Backup Network with ```vSphereBackupNFC``` tag enabled on a NSX setup. @@ -36,7 +35,7 @@ If there is a free physical network adpter on each of the cluster nodes: ## Data Manager Virtual Machine install -The Data Manager Virtual Machine ova can be downloaded from [here](https://vsphere-velero-datamgr.s3-us-west-1.amazonaws.com/datamgr-ob-17253392-photon-3-release-1.1.ova). +The Data Manager Virtual Machine ova can be downloaded from [here](https://github.com/vmware-tanzu/velero-plugin-for-vsphere/releases). It is recommended to power on the Data manager VM after enabling the velero-vsphere service and installing Velero + vSphere plugin in the Supervisor cluster. @@ -48,7 +47,7 @@ It is recommended to power on the Data manager VM after enabling the velero-vsph - guestinfo.cnsdp.vcPassword - guestinfo.cnsdp.vcPort - guestinfo.cnsdp.veleroNamespace - - guestinfo.cnsdp.datamgrImage (if not configured, will use the image from dockerhub vsphereveleroplugin/data-manager-for-plugin:1.1.0) + - guestinfo.cnsdp.datamgrImage - guestinfo.cnsdp.updateKubectl (default false, to avoid kubectl from wcp master on every VM restart) 3. Power On the Data Manager VM diff --git a/docs/supervisor-deploy-with-image-from-private-registry.md b/docs/supervisor-deploy-with-image-from-private-registry.md new file mode 100644 index 00000000..21e510aa --- /dev/null +++ b/docs/supervisor-deploy-with-image-from-private-registry.md @@ -0,0 +1,209 @@ +* [Overview](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-Overview) +* [Manifest](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-Manifest) +* [Deploy Velero Operator](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-DeployVeleroOperator) + * [Add Velero vSphere Operator as a vSphere Service](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-AddVelerovSphereOperatorasavSphereService) + * [Install Velero vSphere Operator on Supervisor Cluster(s)](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-InstallVelerovSphereOperatoronSupervisorCluster(s)) + * [Successful Deployment of Velero vSphere Operator](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-SuccessfulDeploymentofVelerovSphereOperator) +* [Deploy Velero Instance](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-DeployVeleroInstance) +* [Deploy Data Manager VM](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-DeployDataManagerVM) + * [Create a Supervisor Namespace for Velero](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-CreateaSupervisorNamespaceforVelero) + * [Configure Permission for Supervisor DevOps](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-ConfigurePermissionforSupervisorDevOps) + * [Deploy Velero and Plugins](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-DeployVeleroandPlugins) + * [Images From the Same Private Registry](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-ImagesFromtheSamePrivateRegistry) + * [Images From Any Public Registry](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-ImagesFromAnyPublicRegistry) + * [Successful Deployment of Velero](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-SuccessfulDeploymentofVelero) + * [vSphere UI](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-vSphereUI) + * [Kubectl CLI](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-KubectlCLI) +* [Reference](#DeployVeleroinSupervisorClusterwithImagesfromPrivateRegistry-Reference) + +Overview +======== + +In Air-gapped environment, users usually would like to deploy application from private registry which is protected by registry credential. To meet this requirement, we support deploying Velero in Supervisor Cluster with images from private registry. + +Manifest +======== + +* Velero vSphere operator YAML: [https://vmwaresaas.jfrog.io/ui/api/v1/download?repoKey=Velero-YAML&path=Velero%252FSupervisorService%252F1.0.0%252Fvelero-supervisorservice-1.0.0.yaml](https://vmwaresaas.jfrog.io/ui/api/v1/download?repoKey=Velero-YAML&path=Velero%252FSupervisorService%252F1.0.0%252Fvelero-supervisorservice-1.0.0.yaml) + +Note: The following screenshots show the installation using Velero vSphere Operator 1.0.0. If you want to install using the latest Velero vSphere Operator version, refer to this [page](https://github.com/vsphere-tmm/Supervisor-Services/blob/main/README.md#velero-versions). + +Deploy Velero Operator +====================== + +Add Velero vSphere Operator as a vSphere Service +------------------------------------------------ + +Navigate to Workload Management Services page. Click the botton in red box below to add a new service in vSphere Services. (It can be done by corresponding DCLI command as well) + +![](supervisor-private-registry-add-service-1.png) + + + +Upload the Velero vSphere operator YAML mentioned in **Manifest** section above. + +![](supervisor-private-registry-add-service-2.png)![](supervisor-private-registry-add-service-3.png) + + + +Install Velero vSphere Operator on Supervisor Cluster(s) +-------------------------------------------------------- + +Select "Install on Supervisors" as below to install the newly added vSphere Service, Velero vSphere Operator, on Supervisor Cluster(s). + +![](supervisor-private-registry-install-velero-operator-1.png) + + + +Provide configuration parameters for install Velero vSphere Operator on Supervisor Clusters. + +**Note**: To deploy velero operator with images from private registry, both **registryUsername** and **registryPasswd** are required. Please make sure Velero vSphere Operator image with **Select Version** is available in the registry as specified in the **registryName** field. + +![](supervisor-private-registry-install-velero-operator-2.png) + +The above screenshot shows how add Key Value Pairs of the registry while installing Velero vSphere Operator on Supervisor Cluster as in vCenter 7.  + +In VC 8.0, the option will not show as Key Value Pairs as VC 7.0. Customers can provide those options by entering them in "YAML Service Config", see the following example. + +![](supervisor-private-registry-install-velero-operator-3.png) + +Successful Deployment of Velero vSphere Operator +------------------------------------------------ + +After Velero vSphere Operator is installed in Supervisor Cluster, we can see that a new vSphere Service instance for Velero vSphere Operator is added to Supervisor Cluster and its Service Status is in the **Configured** state. + +![](supervisor-private-registry-install-velero-operator-4.png)![](supervisor-private-registry-install-velero-operator-5.png) + + + +Meanwhile, we can see Velero vSphere operator is in **Running** status in its own Supervisor namespace (svc-velero-vsphere-domain-c8 in this case). + +![](supervisor-private-registry-install-velero-operator-6.png) + +That's it for deploying Velero vSphere Operator in Supervisor Cluster with image from private registry. + +Deploy Velero Instance +====================== + +Deploy Data Manager VM +====================== + +Please refer to [https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/main/docs/supervisor-datamgr.md](https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/main/docs/supervisor-datamgr.md). + +Create a Supervisor Namespace for Velero +---------------------------------------- + +![](supervisor-private-registry-deploy-data-manager-1.png) + +![](supervisor-private-registry-deploy-data-manager-2.png) + +Configure Permission for Supervisor DevOps +------------------------------------------ + +Supervisor DevOps will need EDIT permission to deploy Velero in the Supervisor Namespace. It is **optional** for vSphere Admin. + +![](supervisor-private-registry-add-permission-1.png) + +![](supervisor-private-registry-add-permission-2.png) + +Deploy Velero and Plugins +------------------------- + +It is supported to deploy Velero and plugins with images in the following two cases. + +### Images From the Same Private Registry + +**Note**: To deploy Velero with images from private registry as well, it would only work for images from the same private registry whose credential has been provided while enable Velero vSphere Operator in Supervisor Cluster. Also, the **\--use-private-registry** option in the install command is required. + +Below is an example of velero-vsphere install command. + +```bash +BUCKET=velero-v1.5.1-backups +REGION=minio +NAMESPACE=velero +S3URL=http://csm-minio.eng.vmware.com + +VSPHERE_PLUGIN_IMAGE=harbor-stg-repo.vmware.com/velerovsphereoperator/velero-plugin-for-vsphere:v1.3.1 +AWS_PLUGIN_IMAGE=harbor-stg-repo.vmware.com/velerovsphereoperator/velero-plugin-for-aws:v1.1.0 +VELERO_IMAGE=harbor-stg-repo.vmware.com/velerovsphereoperator/velero:v1.5.1 + +# install velero in air-gapped environment +velero-vsphere install \ + --namespace $NAMESPACE \ + --image $VELERO_IMAGE \ + --provider aws \ + --plugins $AWS_PLUGIN_IMAGE,$VSPHERE_PLUGIN_IMAGE \ + --bucket $BUCKET \ + --secret-file ~/.minio/credentials \ + --snapshot-location-config region=$REGION \ + --backup-location-config region=$REGION,s3ForcePathStyle="true",s3Url=$S3URL \ + --use-private-registry # <====== Key Option to deploy Velero with images from private registry +``` + + + +Instead, the following two cases would end up with **ImagePullBackOff** error below. + +* Deploying velero with images from a different private registry. +* Deploying velero with images from the same private registry, but without using the **\--use-private-registry** option in the install command above. + +```bash +0s Warning Failed pod/backup-driver-5c585967c9-dbm76 Failed to pull image "harbor-stg-repo.vmware.com/velerovsphereoperator/backup-driver:v1.3.1": rpc error: code = Unknown desc = failed to pull and unpack image "harbor-stg-repo.vmware.com/velerovsphereoperator/backup-driver:v1.3.1": failed to resolve reference "harbor-stg-repo.vmware.com/velerovsphereoperator/backup-driver:v1.3.1": unexpected status code [manifests v1.3.1]: 401 Unauthorized +``` + +### Images From Any Public Registry + +Alternatively, it is also OK to deploy velero instance with images from any other public registry, even if the Velero vSphere Operator is deployed with images from a private registry. + +Below is an example of velero-vsphere install command. + +```bash +BUCKET=velero-v1.5.1-backups +REGION=minio +NAMESPACE=velero +S3URL=http://csm-minio.eng.vmware.com + +VSPHERE_PLUGIN_IMAGE=harbor-repo.vmware.com/velero/velero-plugin-for-vsphere:v1.3.1 +AWS_PLUGIN_IMAGE=harbor-repo.vmware.com/velero/velero-plugin-for-aws:v1.1.0 +VELERO_IMAGE=harbor-repo.vmware.com/velero/velero:v1.5.1 + +# install velero in air-gapped environment +velero-vsphere install \ + --namespace $NAMESPACE \ + --image $VELERO_IMAGE \ + --provider aws \ + --plugins $AWS_PLUGIN_IMAGE,$VSPHERE_PLUGIN_IMAGE \ + --bucket $BUCKET \ + --secret-file ~/.minio/credentials \ + --snapshot-location-config region=$REGION \ + --backup-location-config region=$REGION,s3ForcePathStyle="true",s3Url=$S3URL +``` + + + +Successful Deployment of Velero +------------------------------- + +Below are observations from both vSphere UI and CLI when Velero and plugins are deployed on Supervisor Cluster sucessfully. + +### vSphere UI + +![](supervisor-private-registery-deploy-success.png) + +### Kubectl CLI + +```bash +$ kubectl -n velero get all +NAME READY STATUS RESTARTS AGE +pod/backup-driver-cb4d96d57-glxfz 1/1 Running 0 4m30s +pod/velero-744cfc7ccc-gn6cn 1/1 Running 0 4m47s + +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/backup-driver 1/1 1 1 4m30s +deployment.apps/velero 1/1 1 1 4m48s + +NAME DESIRED CURRENT READY AGE +replicaset.apps/backup-driver-cb4d96d57 1 1 1 4m30s +replicaset.apps/velero-744cfc7ccc 1 1 1 4m48s +``` + \ No newline at end of file diff --git a/docs/supervisor-private-registery-deploy-success.png b/docs/supervisor-private-registery-deploy-success.png new file mode 100644 index 00000000..7c03ab62 Binary files /dev/null and b/docs/supervisor-private-registery-deploy-success.png differ diff --git a/docs/supervisor-private-registry-add-permission-1.png b/docs/supervisor-private-registry-add-permission-1.png new file mode 100644 index 00000000..0e99fb2a Binary files /dev/null and b/docs/supervisor-private-registry-add-permission-1.png differ diff --git a/docs/supervisor-private-registry-add-permission-2.png b/docs/supervisor-private-registry-add-permission-2.png new file mode 100644 index 00000000..144f7c4b Binary files /dev/null and b/docs/supervisor-private-registry-add-permission-2.png differ diff --git a/docs/supervisor-private-registry-add-service-1.png b/docs/supervisor-private-registry-add-service-1.png new file mode 100644 index 00000000..e23a9cac Binary files /dev/null and b/docs/supervisor-private-registry-add-service-1.png differ diff --git a/docs/supervisor-private-registry-add-service-2.png b/docs/supervisor-private-registry-add-service-2.png new file mode 100644 index 00000000..b670d4a4 Binary files /dev/null and b/docs/supervisor-private-registry-add-service-2.png differ diff --git a/docs/supervisor-private-registry-add-service-3.png b/docs/supervisor-private-registry-add-service-3.png new file mode 100644 index 00000000..5d671762 Binary files /dev/null and b/docs/supervisor-private-registry-add-service-3.png differ diff --git a/docs/supervisor-private-registry-deploy-data-manager-1.png b/docs/supervisor-private-registry-deploy-data-manager-1.png new file mode 100644 index 00000000..7242ae37 Binary files /dev/null and b/docs/supervisor-private-registry-deploy-data-manager-1.png differ diff --git a/docs/supervisor-private-registry-deploy-data-manager-2.png b/docs/supervisor-private-registry-deploy-data-manager-2.png new file mode 100644 index 00000000..fc5f6b33 Binary files /dev/null and b/docs/supervisor-private-registry-deploy-data-manager-2.png differ diff --git a/docs/supervisor-private-registry-install-velero-operator-1.png b/docs/supervisor-private-registry-install-velero-operator-1.png new file mode 100644 index 00000000..ea50afe5 Binary files /dev/null and b/docs/supervisor-private-registry-install-velero-operator-1.png differ diff --git a/docs/supervisor-private-registry-install-velero-operator-2.png b/docs/supervisor-private-registry-install-velero-operator-2.png new file mode 100644 index 00000000..c7f17efd Binary files /dev/null and b/docs/supervisor-private-registry-install-velero-operator-2.png differ diff --git a/docs/supervisor-private-registry-install-velero-operator-3.png b/docs/supervisor-private-registry-install-velero-operator-3.png new file mode 100644 index 00000000..8ff9cc17 Binary files /dev/null and b/docs/supervisor-private-registry-install-velero-operator-3.png differ diff --git a/docs/supervisor-private-registry-install-velero-operator-4.png b/docs/supervisor-private-registry-install-velero-operator-4.png new file mode 100644 index 00000000..4971efe5 Binary files /dev/null and b/docs/supervisor-private-registry-install-velero-operator-4.png differ diff --git a/docs/supervisor-private-registry-install-velero-operator-5.png b/docs/supervisor-private-registry-install-velero-operator-5.png new file mode 100644 index 00000000..9084406f Binary files /dev/null and b/docs/supervisor-private-registry-install-velero-operator-5.png differ diff --git a/docs/supervisor-private-registry-install-velero-operator-6.png b/docs/supervisor-private-registry-install-velero-operator-6.png new file mode 100644 index 00000000..e7b8b560 Binary files /dev/null and b/docs/supervisor-private-registry-install-velero-operator-6.png differ diff --git a/docs/supervisor.md b/docs/supervisor.md index 08668d76..35922a4c 100644 --- a/docs/supervisor.md +++ b/docs/supervisor.md @@ -13,8 +13,9 @@ | Velero Plugin for vSphere Version | vSphere Version | Kubernetes Version | vSphere CSI Driver Version | Velero Version | Velero vSphere Operator Version | Data Manager Version | vSphere Plugin Deprecated | vSphere Plugin EOL Date | |-----------------------------------|---------------------|-------------------------------------------------------------------|----------------------------|----------------|---------------------------------|---------------|------------|---------------| +| 1.5.1 | 8.0U1 | Bundled with vSphere (1.24) | Bundled with vSphere | 1.10.2 | 1.4.0 | 1.2.0 | No | N/A | +| 1.5.1 | 8.0c | Bundled with vSphere (1.24) | Bundled with vSphere | 1.10.2 | 1.4.0 | 1.2.0 | No | N/A | | 1.4.2 | 8.0 | Bundled with vSphere (1.22-1.23) | Bundled with vSphere | 1.9.2 | 1.3.0 | 1.2.0 | No | N/A | -| 1.4.2 | 7.0U3h | Bundled with vSphere (1.22) | Bundled with vSphere | 1.9.2 | 1.3.0 | 1.2.0 | No | N/A | | 1.4.0 | 7.0U3e/f/h | Bundled with vSphere (1.22) | Bundled with vSphere | 1.8.1 | 1.2.0 | 1.1.0 | No | N/A | | 1.3.1 | 7.0U1c/P02 - 7.0U3d | Bundled with vSphere (1.16-1.19, 1.18-1.20, 1.19-1.21) | Bundled with vSphere | 1.5.1 | 1.1.0 | 1.1.0 | No | N/A | | 1.3.0 | 7.0U1c/P02 - 7.0U3d | Bundled with vSphere (1.16-1.19, 1.18-1.20, 1.19-1.21) | Bundled with vSphere | 1.5.1 | 1.1.0 | 1.1.0 | Yes | December 2022 | diff --git a/docs/vanilla.md b/docs/vanilla.md index 1d008ddc..bc95a146 100644 --- a/docs/vanilla.md +++ b/docs/vanilla.md @@ -14,6 +14,8 @@ | Velero Plugin for vSphere Version | vSphere Version | Kubernetes Version | vSphere CSI Driver Version | Velero Version | vSphere Plugin Deprecated | vSphere Plugin EOL Date | |-----------------------------------|------------------------|--------------------|-----------------------------------|----------------|------------|---------------| +1.5.1 | 8.0U2 | 1.27 | 3.0.1 | 1.11.1 | No | N/A | + 1.5.1 | 8.0U1 | 1.26-1.27 | 3.0.1 | 1.10.2, 1.11.1 | No | N/A | | 1.4.2 | 8.0 | 1.24-1.25 | 2.7.0 | 1.9.2 | No | N/A | | 1.4.2 | 7.0U3h | 1.24-1.25 | 2.7.0 | 1.9.2 | No | N/A | | 1.4.1 | 8.0 | 1.24-1.25 | 2.7.0 | 1.9.2 | No | N/A | @@ -280,6 +282,7 @@ Snapshot CRD has a number of phases for the `.status.phase` field: * Canceling: the upload of snapshot is being cancelled * Canceled: the upload of snapshot is cancelled * CleanupAfterUploadFailed: the Cleanup of local snapshot after the upload of snapshot was failed +* UploadFailedAfterRetry: the snapshot is failed to be uploaded after retries, and local snapshot is deleted #### Uploads @@ -330,6 +333,7 @@ Upload CRD has a number of phases for the `.status.phase` field: * CleanupFailed: delete local snapshot failed after the upload, this case will also be retried * Canceling: upload is being cancelled. It would happen if `velero backup delete` is called while the upload of snapshot is in progress. * Canceled: upload is cancelled. +* UploadFailedAfterRetry: Upload failed after retries and local snapshot is deleted. UploadError uploads will be periodically retried. At that point their phase will return to InProgress. After an upload has been successfully completed, its record will remain for a period of time and eventually be removed. diff --git a/go.mod b/go.mod index 900cfb25..c24c4552 100644 --- a/go.mod +++ b/go.mod @@ -132,14 +132,14 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect - golang.org/x/crypto v0.6.0 // indirect + golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20210916165020-5cb4fee858ee // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index b48de52f..6e6df348 100644 --- a/go.sum +++ b/go.sum @@ -955,8 +955,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1058,8 +1058,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1187,13 +1187,13 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1204,8 +1204,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/hack/update-generated-crd-code.sh b/hack/update-generated-crd-code.sh index 0f010446..73e5061d 100755 --- a/hack/update-generated-crd-code.sh +++ b/hack/update-generated-crd-code.sh @@ -23,8 +23,10 @@ if [[ -z "${GOPATH}" ]]; then GOPATH=~/go fi +# Please make sure you install k8s.io/code-generator@v0.18.8, you may hit failure when running this script if you install k8s.io/code-generator with different version if [[ ! -d "${GOPATH}/src/k8s.io/code-generator" ]]; then echo "k8s.io/code-generator missing from GOPATH" + echo "please retry after installing k8s.io/code-generator@v0.18.8" exit 1 fi diff --git a/pkg/apis/backupdriver/v1alpha1/snapshot.go b/pkg/apis/backupdriver/v1alpha1/snapshot.go index 9d3525b0..3c24363c 100644 --- a/pkg/apis/backupdriver/v1alpha1/snapshot.go +++ b/pkg/apis/backupdriver/v1alpha1/snapshot.go @@ -21,27 +21,33 @@ type SnapshotSpec struct { // New - No work yet, next phase is InProgress // InProgress - snapshot being taken // Snapshotted - local snapshot complete, next phase is Protecting or SnapshotFailed -// SnapshotFailed - end state, snapshot was not able to be taken +// SnapshotFailed - terminal state, snapshot was not able to be taken // Uploading - snapshot is being moved to durable storage -// Uploaded - end state, snapshot has been protected -// UploadFailed - end state, unable to move to durable storage +// Uploaded - terminal state, snapshot has been protected +// UploadFailed - unable to move to durable storage // Canceling - when the SanpshotCancel flag is set, if the Snapshot has not already moved into a terminal state, the // status will move to Canceling. The snapshot ID will be removed from the status status if has been filled in // and the snapshot ID will not longer be valid for a Clone operation // Canceled - the operation was canceled, the snapshot ID is not valid +// CleanupAfterUploadFailed - terminal state, unable to delete local snapshot +// UploadFailedAfterRetry - terminal state, local snapshot is deleted after upload CR retries reach the maximum retry count. User can set the "upload-cr-retry-max" +// parameter in config map velero-vsphere-plugin-config to specify the maximum count the upload CR will retry before trying to delete +// the local snapshot. If user does not specify this in the config map, default retry count is set to 10. + type SnapshotPhase string const ( - SnapshotPhaseNew SnapshotPhase = "New" - SnapshotPhaseInProgress SnapshotPhase = "InProgress" - SnapshotPhaseSnapshotted SnapshotPhase = "Snapshotted" - SnapshotPhaseSnapshotFailed SnapshotPhase = "SnapshotFailed" - SnapshotPhaseUploading SnapshotPhase = "Uploading" - SnapshotPhaseUploaded SnapshotPhase = "Uploaded" - SnapshotPhaseUploadFailed SnapshotPhase = "UploadFailed" - SnapshotPhaseCanceling SnapshotPhase = "Canceling" - SnapshotPhaseCanceled SnapshotPhase = "Canceled" - SnapshotPhaseCleanupFailed SnapshotPhase = "CleanupAfterUploadFailed" + SnapshotPhaseNew SnapshotPhase = "New" + SnapshotPhaseInProgress SnapshotPhase = "InProgress" + SnapshotPhaseSnapshotted SnapshotPhase = "Snapshotted" + SnapshotPhaseSnapshotFailed SnapshotPhase = "SnapshotFailed" + SnapshotPhaseUploading SnapshotPhase = "Uploading" + SnapshotPhaseUploaded SnapshotPhase = "Uploaded" + SnapshotPhaseUploadFailed SnapshotPhase = "UploadFailed" + SnapshotPhaseCanceling SnapshotPhase = "Canceling" + SnapshotPhaseCanceled SnapshotPhase = "Canceled" + SnapshotPhaseCleanupFailed SnapshotPhase = "CleanupAfterUploadFailed" + SnapshotPhaseUploadFailedAfterRetry SnapshotPhase = "UploadFailedAfterRetry" ) // UploadOperationProgress represents the progress of a @@ -149,7 +155,7 @@ type CloneFromSnapshotSpec struct { New - No work yet, next phase is InProgress InProgress - snapshot being taken Completed - new object has been created - Failed - end state, clone failed, no new object was created + Failed - terminal state, clone failed, no new object was created Canceling - when the Clone flag is set, if the Clone has not already moved into a terminal state, the status will move to Canceling. The object that was being created will be removed Canceled - the Clone was canceled, no new object was created diff --git a/pkg/apis/backupdriver/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/backupdriver/v1alpha1/zz_generated.deepcopy.go index af3bb8ac..c8b2371b 100644 --- a/pkg/apis/backupdriver/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/backupdriver/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/pkg/apis/datamover/v1alpha1/upload.go b/pkg/apis/datamover/v1alpha1/upload.go index bd81b4db..0e828a40 100644 --- a/pkg/apis/datamover/v1alpha1/upload.go +++ b/pkg/apis/datamover/v1alpha1/upload.go @@ -44,17 +44,28 @@ type UploadSpec struct { } // UploadPhase represents the lifecycle phase of a Upload. -// +kubebuilder:validation:Enum=New;InProgress;Completed;UploadError;CleanupFailed;Canceled;Canceling; +// +kubebuilder:validation:Enum=New;InProgress;Completed;UploadError;CleanupFailed;Canceled;Canceling;UploadFailedAfterRetry; type UploadPhase string +// New: not processed yet +// InProgress: upload is in progress. +// Completed: upload is completed. +// UploadError: upload is failed and will be periodically retried. The status will change to "InProgress" at that point. +// CleanupFailed: delete local snapshot failed after the upload succeed or upload failed after maximum retry, this case will retry the delete local snapshot. +// Canceling: upload is being cancelled. It would happen if `velero backup delete` is called while the upload of snapshot is in progress. +// Canceled: upload is cancelled. +// UploadFailedAfterRetry: terminal state. Upload failed after retry and local snapshot is deleted. User can set the "upload-cr-retry-max" +// parameter in config map velero-vsphere-plugin-config to specify the maximum count that the upload CR will retry. +// If user does not specify this in the config map, default retry count is set to 10. const ( - UploadPhaseNew UploadPhase = "New" - UploadPhaseInProgress UploadPhase = "InProgress" - UploadPhaseCompleted UploadPhase = "Completed" - UploadPhaseUploadError UploadPhase = "UploadError" - UploadPhaseCleanupFailed UploadPhase = "CleanupFailed" - UploadPhaseCanceling UploadPhase = "Canceling" - UploadPhaseCanceled UploadPhase = "Canceled" + UploadPhaseNew UploadPhase = "New" + UploadPhaseInProgress UploadPhase = "InProgress" + UploadPhaseCompleted UploadPhase = "Completed" + UploadPhaseUploadError UploadPhase = "UploadError" + UploadPhaseCleanupFailed UploadPhase = "CleanupFailed" + UploadPhaseCanceling UploadPhase = "Canceling" + UploadPhaseCanceled UploadPhase = "Canceled" + UploadPhaseUploadFailedAfterRetry UploadPhase = "UploadFailedAfterRetry" ) // UploadStatus is the current status of a Upload. diff --git a/pkg/apis/datamover/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/datamover/v1alpha1/zz_generated.deepcopy.go index ef65765d..e5051641 100644 --- a/pkg/apis/datamover/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/datamover/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/pkg/backupdriver/backup_driver_controller_base.go b/pkg/backupdriver/backup_driver_controller_base.go index 02b56e42..f48305e3 100644 --- a/pkg/backupdriver/backup_driver_controller_base.go +++ b/pkg/backupdriver/backup_driver_controller_base.go @@ -18,13 +18,14 @@ package backupdriver import ( "context" + "strings" + "time" + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/backuprepository" "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/constants" "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/utils" corev1 "k8s.io/api/core/v1" v1 "k8s.io/client-go/informers/core/v1" - "strings" - "time" "k8s.io/client-go/rest" @@ -957,6 +958,8 @@ func (ctrl *backupDriverController) syncUploadByKey(key string) error { newSnapshotStatusPhase = backupdriverapi.SnapshotPhaseUploadFailed case datamoverapi.UploadPhaseCleanupFailed: newSnapshotStatusPhase = backupdriverapi.SnapshotPhaseCleanupFailed + case datamoverapi.UploadPhaseUploadFailedAfterRetry: + newSnapshotStatusPhase = backupdriverapi.SnapshotPhaseUploadFailedAfterRetry default: ctrl.logger.Debugf("syncUploadByKey: No change needed for upload phase %s", upload.Status.Phase) return nil diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index bde74bbf..ab9e6444 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -178,6 +178,14 @@ const ( const ( RetryInterval = 5 RetryMaximum = 5 + // Default maximum retry count for upload CR. + // After upload CR retry reaches the maximum and still cannot upload the snapshot to durable storage. Upload CR will stop further retry and + // try to delete the local snapshot. + // If local snapshot is deleted successfully, the status of upload CR will change to "UploadFailedAfterRetry". + // If local snapshot cannot be deleted, the status of upload CR will change to "CleanupFailed". + // If the default maximum retry count does work for user, it can be overwritten by setting the "upload-cr-retry-max" + // parameter in config map velero-vsphere-plugin-config. + DefaultUploadCRRetryMaximum = 10 ) // Keys for supervisor cluster parameters @@ -255,9 +263,9 @@ var ResourcesToBlock = map[string]bool{ // We comment the NetworkAttachmentDefinition resource out as it is not a vSphere specific resource //"network-attachment-definitions.k8s.cni.cncf.io": true, // real name of NetworkAttachmentDefinition //"networkattachmentdefinitions.k8s.cni.cncf.io": true, // parsed name of NetworkAttachmentDefinition - "networkinterfaces.netoperator.vmware.com": true, - "networks.netoperator.vmware.com": true, - "nsxerrors.nsx.vmware.com": true, + "networkinterfaces.netoperator.vmware.com": true, + "networks.netoperator.vmware.com": true, + "nsxerrors.nsx.vmware.com": true, //"nsxlbmonitors.vmware.com": true, // DO NOT ADD IT BACK //"nsxloadbalancermonitors.vmware.com": true, // DO NOT ADD IT BACK "nsxlocks.nsx.vmware.com": true, @@ -399,7 +407,7 @@ const ( const VsphereVolumeSnapshotLocationProvider = "velero.io/vsphere" const ( - VSphereCSIDriverName = "csi.vsphere.vmware.com" + VSphereCSIDriverName = "csi.vsphere.vmware.com" VSphereCSIDriverMigrationAnnotation = "pv.kubernetes.io/migrated-to" ) @@ -418,12 +426,13 @@ const ( VSphereSecretNameKey = "vsphere_secret_name" DefaultSecretName = "velero-vsphere-config-secret" DefaultSecretNamespace = "velero" + UploadCRRetryMaximumKey = "upload-cr-retry-max" ) const ( // AnnVolumeHealth is the key for HealthStatus annotation on volume claim // for vSphere CSI Driver. AnnVolumeHealth = "volumehealth.storage.kubernetes.io/health" - // key for expressing timestamp for volume health annotation - AnnVolumeHealthTS = "volumehealth.storage.kubernetes.io/health-timestamp" + // key for expressing timestamp for volume health annotation + AnnVolumeHealthTS = "volumehealth.storage.kubernetes.io/health-timestamp" ) diff --git a/pkg/controller/upload_controller.go b/pkg/controller/upload_controller.go index 3f2e3b6e..dc792a27 100644 --- a/pkg/controller/upload_controller.go +++ b/pkg/controller/upload_controller.go @@ -19,12 +19,13 @@ package controller import ( "context" "fmt" + "math" + "time" + backupdriverapi "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/apis/backupdriver/v1alpha1" "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/backuprepository" "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/constants" "k8s.io/apimachinery/pkg/util/wait" - "math" - "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -292,17 +293,22 @@ func (c *uploadController) processUpload(req *pluginv1api.Upload) error { }).Debug("Upload request updated by retrieving from kubernetes API server") if req.Status.Phase == pluginv1api.UploadPhaseCanceled { - log.WithField("phase", req.Status.Phase).WithField("generation", req.Generation).Debug("The status of upload CR in kubernetes API server is canceled. Skipping it") + log.WithField("phase", req.Status.Phase).WithField("generation", req.Generation).Debug("The status of Upload CR in Kubernetes API server is canceled. Skipping it") return nil } if req.Status.Phase == pluginv1api.UploadPhaseCanceling { - log.WithField("phase", req.Status.Phase).WithField("generation", req.Generation).Debug("The status of upload CR in kubernetes API server is canceling. Skipping it") + log.WithField("phase", req.Status.Phase).WithField("generation", req.Generation).Debug("The status of Upload CR in Kubernetes API server is canceling. Skipping it") return nil } if req.Status.Phase == pluginv1api.UploadPhaseCompleted { - log.WithField("phase", req.Status.Phase).WithField("generation", req.Generation).Debug("The status of upload CR in kubernetes API server is completed. Skipping it") + log.WithField("phase", req.Status.Phase).WithField("generation", req.Generation).Debug("The status of Upload CR in Kubernetes API server is completed. Skipping it") + return nil + } + + if req.Status.Phase == pluginv1api.UploadPhaseUploadFailedAfterRetry { + log.WithField("phase", req.Status.Phase).WithField("generation", req.Generation).Debug("The status of Upload CR in Kubernetes API server is failed after retry. Skipping it") return nil } @@ -334,7 +340,7 @@ func (c *uploadController) processUpload(req *pluginv1api.Upload) error { errMsg := fmt.Sprintf("Failed to clean up local snapshot, %s, error: %v. will retry.", peID.String(), errors.WithStack(err)) _, err = c.patchUploadByStatusWithRetry(req, pluginv1api.UploadPhaseCleanupFailed, errMsg) if err != nil { - errMsg = fmt.Sprintf("Failed to patch upload status with retry, errror: %v", errors.WithStack(err)) + errMsg = fmt.Sprintf("Failed to patch Upload status with retry, errror: %v", errors.WithStack(err)) log.Error(errMsg) } return errors.New(errMsg) @@ -372,12 +378,35 @@ func (c *uploadController) processUpload(req *pluginv1api.Upload) error { return nil } else { errMsg := fmt.Sprintf("Failed to upload snapshot, %v, to durable object storage. %v", peID.String(), errors.WithStack(err)) - _, err = c.patchUploadByStatusWithRetry(req, pluginv1api.UploadPhaseUploadError, errMsg) + uploadCRRetryMaximum := utils.GetUploadCRRetryMaximum(nil, c.logger) + log.Infof("UploadCRRetryMaximum is %d", uploadCRRetryMaximum) + if req.Status.RetryCount < int32(uploadCRRetryMaximum) { + _, err = c.patchUploadByStatusWithRetry(req, pluginv1api.UploadPhaseUploadError, errMsg) + if err != nil { + errMsg = fmt.Sprintf("%v. %v", errMsg, errors.WithStack(err)) + } + log.Error(errMsg) + return errors.New(errMsg) + } + log.Infof("Upload CR %s failed to upload snapshot after retrying %d times", req.Name, uploadCRRetryMaximum) + err = c.snapMgr.DeleteLocalSnapshot(peID) + if err != nil { + errMsg := fmt.Sprintf("Failed to clean up local snapshot, %s, error: %v. will retry.", peID.String(), errors.WithStack(err)) + _, err = c.patchUploadByStatusWithRetry(req, pluginv1api.UploadPhaseCleanupFailed, errMsg) + if err != nil { + errMsg = fmt.Sprintf("Failed to patch Upload status with retry, errror: %v", errors.WithStack(err)) + log.Error(errMsg) + } + return errors.New(errMsg) + } + msg := fmt.Sprintf("Successfully deleted local snapshot, %v", peID.String()) + log.Info(msg) + // UploadPhaseUploadFailedAfterRetry is a terminal state, and upload CR will not retry + _, err = c.patchUploadByStatusWithRetry(req, pluginv1api.UploadPhaseUploadFailedAfterRetry, msg) if err != nil { - errMsg = fmt.Sprintf("%v. %v", errMsg, errors.WithStack(err)) + log.Error(err) } - log.Error(errMsg) - return errors.New(errMsg) + return nil } } @@ -487,6 +516,13 @@ func (c *uploadController) patchUploadByStatus(req *pluginv1api.Upload, newPhase r.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} r.Status.Message = msg }) + case pluginv1api.UploadPhaseUploadFailedAfterRetry: + req, err = c.patchUpload(req, func(r *pluginv1api.Upload) { + r.Status.Phase = newPhase + r.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} + r.Status.Message = msg + }) + default: err = errors.New("Unexpected upload phase") } diff --git a/pkg/controller/upload_controller_test.go b/pkg/controller/upload_controller_test.go index 5ec3697c..aebc7e17 100644 --- a/pkg/controller/upload_controller_test.go +++ b/pkg/controller/upload_controller_test.go @@ -20,7 +20,13 @@ import ( "context" "encoding/json" "errors" + "reflect" + "strconv" + "testing" + "time" + "github.com/agiledragon/gomonkey" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmware-tanzu/astrolabe/pkg/astrolabe" @@ -32,16 +38,14 @@ import ( informers "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/generated/informers/externalversions" "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/snapshotmgr" veleroplugintest "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/test" + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" kubefake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" core "k8s.io/client-go/testing" "k8s.io/utils/clock" - "reflect" - "strconv" - "testing" - "time" ) func defaultUpload() *builder.UploadBuilder { @@ -463,20 +467,20 @@ func TestProcessUploadCleanupFailed(t *testing.T) { expectedPhase v1.UploadPhase expectedErr bool retry int32 - } { + }{ { - name: "If delete local snapshot successfully, change upload phase to UploadPhaseCompleted and return without error", - key: "velero/upload-1", - upload: defaultUpload().Phase(v1.UploadPhaseCleanupFailed).SnapshotID("ivd:1234:1234").Retry(0).Result(), + name: "If delete local snapshot successfully, change upload phase to UploadPhaseCompleted and return without error", + key: "velero/upload-1", + upload: defaultUpload().Phase(v1.UploadPhaseCleanupFailed).SnapshotID("ivd:1234:1234").Retry(0).Result(), expectedPhase: v1.UploadPhaseCompleted, - expectedErr: false, + expectedErr: false, }, { - name: "If fail to delete local snapshot, keep upload phase as UploadPhaseCleanupFailed and return error", - key: "velero/upload-1", - upload: defaultUpload().Phase(v1.UploadPhaseCleanupFailed).SnapshotID("ivd:1234:1234").Retry(0).Result(), + name: "If fail to delete local snapshot, keep upload phase as UploadPhaseCleanupFailed and return error", + key: "velero/upload-1", + upload: defaultUpload().Phase(v1.UploadPhaseCleanupFailed).SnapshotID("ivd:1234:1234").Retry(0).Result(), expectedPhase: v1.UploadPhaseCleanupFailed, - expectedErr: true, + expectedErr: true, retry: constants.MIN_RETRY, }, } @@ -531,4 +535,120 @@ func TestProcessUploadCleanupFailed(t *testing.T) { } }) } -} \ No newline at end of file +} + +func TestUploadFailedAfterRetry(t *testing.T) { + tests := []struct { + name string + key string + retry int32 + upload *v1.Upload + expectedPhase v1.UploadPhase + expectedErr error + expectedRetry int32 + }{ + { + name: "Test retry for upload with uploaderror until upload-cr-retry-max reached", + key: "velero/upload-1", + retry: constants.MIN_RETRY, + upload: defaultUpload().Phase(v1.UploadPhaseInProgress).SnapshotID("ivd:1234:1234").Retry(0).Result(), + expectedPhase: v1.UploadPhaseUploadFailedAfterRetry, + expectedErr: errors.New("Failed to upload snapshot, ivd:1234:1234, to durable object storage. Failed at copying to remote repository"), + expectedRetry: 5, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var ( + client = fake.NewSimpleClientset(test.upload) + sharedInformers = informers.NewSharedInformerFactory(client, 0) + logger = veleroplugintest.NewLogger() + kubeClient = kubefake.NewSimpleClientset() + ) + + c := &uploadController{ + genericController: newGenericController("upload-test", logger), + kubeClient: kubeClient, + uploadClient: client.DatamoverV1alpha1(), + uploadLister: sharedInformers.Datamover().V1alpha1().Uploads().Lister(), + nodeName: "upload-test", + clock: &clock.RealClock{}, + dataMover: &dataMover.DataMover{}, + snapMgr: &snapshotmgr.SnapshotManager{}, + } + + // First time set Inprogress to UploadError + require.NoError(t, sharedInformers.Datamover().V1alpha1().Uploads().Informer().GetStore().Add(test.upload)) + patches := gomonkey.ApplyMethod(reflect.TypeOf(c.dataMover), "CopyToRepo", func(_ *dataMover.DataMover, _ astrolabe.ProtectedEntityID) (astrolabe.ProtectedEntityID, error) { + return astrolabe.ProtectedEntityID{}, errors.New("Failed at copying to remote repository") + }) + defer patches.Reset() + + patches.ApplyFunc(utils.GetUploadCRRetryMaximum, func(_ *rest.Config, _ logrus.FieldLogger) int { + return 5 + }) + + c.processUploadFunc = c.processUpload + err := c.processUploadItem(test.key) + require.Equal(t, err.Error(), test.expectedErr.Error()) + + // Second time set Inprogress to UploadError + require.NoError(t, sharedInformers.Datamover().V1alpha1().Uploads().Informer().GetStore().Add(test.upload)) + patches.ApplyMethod(reflect.TypeOf(c.dataMover), "CopyToRepo", func(_ *dataMover.DataMover, _ astrolabe.ProtectedEntityID) (astrolabe.ProtectedEntityID, error) { + return astrolabe.ProtectedEntityID{}, errors.New("Failed at copying to remote repository") + }) + + c.processUploadFunc = c.processUpload + err = c.processUploadItem(test.key) + require.Equal(t, err.Error(), test.expectedErr.Error()) + + // Third time set Inprogress to UploadError + require.NoError(t, sharedInformers.Datamover().V1alpha1().Uploads().Informer().GetStore().Add(test.upload)) + patches.ApplyMethod(reflect.TypeOf(c.dataMover), "CopyToRepo", func(_ *dataMover.DataMover, _ astrolabe.ProtectedEntityID) (astrolabe.ProtectedEntityID, error) { + return astrolabe.ProtectedEntityID{}, errors.New("Failed at copying to remote repository") + }) + + c.processUploadFunc = c.processUpload + err = c.processUploadItem(test.key) + require.Equal(t, err.Error(), test.expectedErr.Error()) + + // Fourth time set Inprogress to UploadError + require.NoError(t, sharedInformers.Datamover().V1alpha1().Uploads().Informer().GetStore().Add(test.upload)) + patches.ApplyMethod(reflect.TypeOf(c.dataMover), "CopyToRepo", func(_ *dataMover.DataMover, _ astrolabe.ProtectedEntityID) (astrolabe.ProtectedEntityID, error) { + return astrolabe.ProtectedEntityID{}, errors.New("Failed at copying to remote repository") + }) + + c.processUploadFunc = c.processUpload + err = c.processUploadItem(test.key) + require.Equal(t, err.Error(), test.expectedErr.Error()) + + // Fifth time set Inprogress to UploadError + require.NoError(t, sharedInformers.Datamover().V1alpha1().Uploads().Informer().GetStore().Add(test.upload)) + patches.ApplyMethod(reflect.TypeOf(c.dataMover), "CopyToRepo", func(_ *dataMover.DataMover, _ astrolabe.ProtectedEntityID) (astrolabe.ProtectedEntityID, error) { + return astrolabe.ProtectedEntityID{}, errors.New("Failed at copying to remote repository") + }) + + c.processUploadFunc = c.processUpload + err = c.processUploadItem(test.key) + require.Equal(t, err.Error(), test.expectedErr.Error()) + + // Sixth time set Inprogress to UploadFailedAfterRetry + require.NoError(t, sharedInformers.Datamover().V1alpha1().Uploads().Informer().GetStore().Add(test.upload)) + patches.ApplyMethod(reflect.TypeOf(c.dataMover), "CopyToRepo", func(_ *dataMover.DataMover, _ astrolabe.ProtectedEntityID) (astrolabe.ProtectedEntityID, error) { + return astrolabe.ProtectedEntityID{}, errors.New("Failed at copying to remote repository") + }) + + patches.ApplyMethod(reflect.TypeOf(c.snapMgr), "DeleteLocalSnapshot", func(_ *snapshotmgr.SnapshotManager, _ astrolabe.ProtectedEntityID) error { + return nil + }) + + c.processUploadFunc = c.processUpload + err = c.processUploadItem(test.key) + require.Nil(t, err) + res, err := c.uploadClient.Uploads(test.upload.Namespace).Get(context.TODO(), test.upload.Name, metav1.GetOptions{}) + require.Equal(t, test.expectedPhase, res.Status.Phase) + require.Equal(t, test.expectedRetry, res.Status.RetryCount) + }) + } +} diff --git a/pkg/generated/crds/manifests/datamover.cnsdp.vmware.com_uploads.yaml b/pkg/generated/crds/manifests/datamover.cnsdp.vmware.com_uploads.yaml index b606b339..a96f4406 100644 --- a/pkg/generated/crds/manifests/datamover.cnsdp.vmware.com_uploads.yaml +++ b/pkg/generated/crds/manifests/datamover.cnsdp.vmware.com_uploads.yaml @@ -22,10 +22,14 @@ spec: description: Upload describe a velero-plugin backup properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -33,17 +37,22 @@ spec: description: Spec is the custom resource spec properties: backupRepository: - description: BackupRepository provides backup repository info for upload. Used for multiple backup repository. + description: BackupRepository provides backup repository info for + upload. Used for multiple backup repository. type: string backupTimestamp: - description: BackupTimestamp records the time the backup was called. The server's time is used for SnapshotTimestamp + description: BackupTimestamp records the time the backup was called. + The server's time is used for SnapshotTimestamp format: date-time type: string snapshotID: - description: SnapshotID is the identifier for the snapshot of the volume. + description: SnapshotID is the identifier for the snapshot of the + volume. type: string snapshotReference: - description: SnapshotReference is the namespace and snapshot name for this upload request. The format is SnapshotNamespace/SnapshotCRName It is used to update the upload status in the snapshot. + description: SnapshotReference is the namespace and snapshot name + for this upload request. The format is SnapshotNamespace/SnapshotCRName + It is used to update the upload status in the snapshot. type: string uploadCancel: description: UploadCancel indicates request to cancel ongoing upload. @@ -53,19 +62,24 @@ spec: description: UploadStatus is the current status of a Upload. properties: completionTimestamp: - description: CompletionTimestamp records the time an upload was completed. Completion time is recorded even on failed uploads. The server's time is used for CompletionTimestamps + description: CompletionTimestamp records the time an upload was completed. + Completion time is recorded even on failed uploads. The server's + time is used for CompletionTimestamps format: date-time nullable: true type: string currentBackOff: - description: CurrentBackOff records the backoff on retry for failed upload. Retry on upload should obey exponential backoff mechanism. + description: CurrentBackOff records the backoff on retry for failed + upload. Retry on upload should obey exponential backoff mechanism. format: int32 type: integer message: description: Message is a message about the upload's status. type: string nextRetryTimestamp: - description: NextRetryTimestamp should be the timestamp that indicate the next retry for failed upload CR. Used to filter out the upload request which comes in before next retry time. + description: NextRetryTimestamp should be the timestamp that indicate + the next retry for failed upload CR. Used to filter out the upload + request which comes in before next retry time. format: date-time nullable: true type: string @@ -79,12 +93,18 @@ spec: - CleanupFailed - Canceled - Canceling + - UploadFailedAfterRetry type: string processingNode: - description: The DataManager node that has picked up the Upload for processing. This will be updated as soon as the Upload is picked up for processing. If the DataManager couldn't process Upload for some reason it will be picked up by another node. + description: The DataManager node that has picked up the Upload for + processing. This will be updated as soon as the Upload is picked + up for processing. If the DataManager couldn't process Upload for + some reason it will be picked up by another node. type: string progress: - description: Progress holds the total number of bytes of the volume and the current number of backed up bytes. This can be used to display progress information about the backup operation. + description: Progress holds the total number of bytes of the volume + and the current number of backed up bytes. This can be used to display + progress information about the backup operation. properties: bytesDone: format: int64 @@ -94,11 +114,14 @@ spec: type: integer type: object retryCount: - description: RetryCount records the number of retry times for adding a failed Upload which failed due to network issue back to queue. Used for user tracking and debugging. + description: RetryCount records the number of retry times for adding + a failed Upload which failed due to network issue back to queue. + Used for user tracking and debugging. format: int32 type: integer startTimestamp: - description: StartTimestamp records the time an upload was started. The server's time is used for StartTimestamps + description: StartTimestamp records the time an upload was started. + The server's time is used for StartTimestamps format: date-time nullable: true type: string diff --git a/pkg/snapshotmgr/snapshot_manager.go b/pkg/snapshotmgr/snapshot_manager.go index 227d7814..c813bb38 100644 --- a/pkg/snapshotmgr/snapshot_manager.go +++ b/pkg/snapshotmgr/snapshot_manager.go @@ -20,15 +20,16 @@ import ( "context" "encoding/base64" "fmt" + "os" + "strings" + "time" + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/backuprepository" "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/common/vsphere" "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/constants" "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/ivd" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" - "os" - "strings" - "time" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -127,7 +128,7 @@ func NewSnapshotManagerFromConfig(configInfo server.ConfigInfo, s3RepoParams map isLocalMode := utils.GetBool(config[constants.VolumeSnapshotterLocalMode], false) initRemoteStorage := clusterFlavor == constants.VSphere - addOnInits := map[string]server.InitFunc { + addOnInits := map[string]server.InitFunc{ "ivd": ivd.NewIVDProtectedEntityTypeManager, } // Initialize the DirectProtectedEntityManager @@ -542,7 +543,7 @@ func (this *SnapshotManager) isTerminalState(uploadCR *v1api.Upload) bool { if uploadCR == nil { return true } - return uploadCR.Status.Phase == v1api.UploadPhaseCompleted || uploadCR.Status.Phase == v1api.UploadPhaseCleanupFailed || uploadCR.Status.Phase == v1api.UploadPhaseCanceled + return uploadCR.Status.Phase == v1api.UploadPhaseCompleted || uploadCR.Status.Phase == v1api.UploadPhaseCleanupFailed || uploadCR.Status.Phase == v1api.UploadPhaseCanceled || uploadCR.Status.Phase == v1api.UploadPhaseUploadFailedAfterRetry } func (this *SnapshotManager) DeleteLocalSnapshot(peID astrolabe.ProtectedEntityID) error { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index efcba606..97b19f16 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -20,18 +20,19 @@ import ( "context" "encoding/json" "fmt" - "github.com/hashicorp/go-version" - "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/constants" - "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/generated/clientset/versioned/typed/backupdriver/v1alpha1" - "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/ivd" "io/ioutil" - "k8s.io/client-go/tools/clientcmd" "net" "os" "strconv" "strings" "time" + "github.com/hashicorp/go-version" + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/constants" + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/generated/clientset/versioned/typed/backupdriver/v1alpha1" + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/ivd" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/tools/cache" @@ -56,7 +57,6 @@ import ( "k8s.io/client-go/rest" ) - func RetrieveVcConfigSecret(params map[string]interface{}, config *rest.Config, logger logrus.FieldLogger) error { var err error // Declare here to avoid shadowing on using in cluster config only if config == nil { @@ -733,6 +733,51 @@ func GetSecretNamespaceAndName(kubeClient kubernetes.Interface, veleroNs string, return namespace, name } +func GetUploadCRRetryMaximumFromConfig(kubeClient kubernetes.Interface, veleroNs string, configName string, logger logrus.FieldLogger) int { + veleroPuginConfigMap, err := kubeClient.CoreV1().ConfigMaps(veleroNs).Get(context.TODO(), configName, metav1.GetOptions{}) + if err != nil { + logger.Errorf("failed to retrieve %s configuration, err: %+v.\n", configName, err) + return constants.DefaultUploadCRRetryMaximum + } + configMap := veleroPuginConfigMap.Data + if maxRetryCountStr, ok := configMap[constants.UploadCRRetryMaximumKey]; ok { + maxRetryCount, err := strconv.Atoi(maxRetryCountStr) + if err != nil { + logger.Errorf("UploadCRRretryMaximum %s is invalid, err: %+v.\n", maxRetryCountStr, err) + maxRetryCount = constants.DefaultUploadCRRetryMaximum + } + return maxRetryCount + } + return constants.DefaultUploadCRRetryMaximum +} + +func GetUploadCRRetryMaximum(config *rest.Config, logger logrus.FieldLogger) int { + var err error + if config == nil { + config, err = GetKubeClientConfig() + if err != nil { + logger.Errorf("GetKubeClientConfig failed with err: %+v, UploadCRRetryMaximum is set to default: %d", err, constants.DefaultUploadCRRetryMaximum) + return constants.DefaultUploadCRRetryMaximum + } + } + + clientset, err := GetKubeClientSet(config) + if err != nil { + logger.Errorf("GetKubeClienSet failed with err: %+v, UploadCRRetryMaximum is set to default: %d", err, constants.DefaultUploadCRRetryMaximum) + return constants.DefaultUploadCRRetryMaximum + } + + veleroNs, exist := GetVeleroNamespace() + if !exist { + logger.Errorf("The VELERO_NAMESPACE environment variable is empty, assuming velero as namespace\n") + veleroNs = constants.DefaultVeleroNamespace + } + + // Check for velero-vsphere-plugin-config in the velero installation namespace + maxRetryCnt := GetUploadCRRetryMaximumFromConfig(clientset, veleroNs, constants.VeleroVSpherePluginConfig, logger) + return maxRetryCnt +} + /* * Get the configuration to access the Supervisor namespace from the GuestCluster. * This routine will be called only for guest cluster. @@ -1046,7 +1091,7 @@ func GetVcConfigSecretFilterFunc(logger logrus.FieldLogger) func(obj interface{} veleroNs, exist := GetVeleroNamespace() if !exist { - logger.Errorf("RetrieveVcConfigSecret: Failed to lookup the env variable for velero namespace, using " + + logger.Errorf("RetrieveVcConfigSecret: Failed to lookup the env variable for velero namespace, using "+ "default %s", constants.DefaultVeleroNamespace) veleroNs = constants.DefaultVeleroNamespace } diff --git a/pkg/utils/utils_unit_test.go b/pkg/utils/utils_unit_test.go index e8bae4f0..e1a74fef 100644 --- a/pkg/utils/utils_unit_test.go +++ b/pkg/utils/utils_unit_test.go @@ -18,6 +18,10 @@ package utils import ( "errors" + "strings" + "testing" + "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -28,9 +32,6 @@ import ( "k8s.io/client-go/kubernetes" kubeclientfake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/rest" - "strings" - "testing" - "time" "github.com/agiledragon/gomonkey" "github.com/sirupsen/logrus" @@ -118,6 +119,20 @@ var ( "cluster_flavor": "VANILLA", }, } + + pluginConfigVanillaWithUploadCRRetryMax = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.VeleroVSpherePluginConfig, + Namespace: "velero", + }, + Data: map[string]string{ + "cluster_flavor": "VANILLA", + "vsphere_secret_name": "velero-vsphere-config-secret", + "vsphere_secret_namespace": "velero", + "upload-cr-retry-max": "5", + }, + } + pluginConfigGuest = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: constants.VeleroVSpherePluginConfig, @@ -1103,3 +1118,39 @@ func TestRetrieveVcConfigSecret(t *testing.T) { }) } } + +func TestGetUploadCRRetryMaximumFromConfig(t *testing.T) { + tests := []struct { + name string + runtimeObjs []runtime.Object + expectedError error + expectedUploadCRRetryMax int + }{ + { + name: "upload-cr-retry-max is not specified in velero-vsphere-plugin-config", + runtimeObjs: []runtime.Object{ + pluginConfigVanilla, + }, + expectedUploadCRRetryMax: constants.DefaultUploadCRRetryMaximum, + expectedError: nil, + }, + { + name: "upload-cr-retry-max is specified in velero-vsphere-plugin-config", + runtimeObjs: []runtime.Object{ + pluginConfigVanillaWithUploadCRRetryMax, + }, + expectedUploadCRRetryMax: 5, + expectedError: nil, + }, + } + + logger := veleroplugintest.NewLogger() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + kubeClient := kubeclientfake.NewSimpleClientset(test.runtimeObjs...) + actualUploadCRRetryMax := GetUploadCRRetryMaximumFromConfig(kubeClient, constants.DefaultVeleroNamespace, + constants.VeleroVSpherePluginConfig, logger) + assert.Equal(t, test.expectedUploadCRRetryMax, actualUploadCRRetryMax) + }) + } +}