[Cherry-Pick]vCenter password rotation handling. (#233) (#241)

Handling password rotation in plugin.

Testing Done
Verified Password rotation is detected and processed on Vanilla and WCP setup
Tested backup driver and data mgr to verify that the newly rotated password is picked.

Pre-Checkins:
https://container-dp.svc.eng.vmware.com/view/CNS-DP/job/Velero-Pipeline-WCP/191/
https://container-dp.svc.eng.vmware.com/job/Container_Precheck_Velero/670/

Signed-off-by: Deepak Kinni <[email protected]>

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/spf13/cobra v0.0.6
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
-	github.com/vmware-tanzu/astrolabe v0.1.2-0.20201008235446-3a8d26a2c1bf
+	github.com/vmware-tanzu/astrolabe v0.1.2-0.20201114003729-394c01f13c75
 	github.com/vmware-tanzu/velero v1.5.1
 	k8s.io/api v0.18.4
 	k8s.io/apiextensions-apiserver v0.18.4

go.sum

@@ -787,8 +787,8 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
-github.com/vmware-tanzu/astrolabe v0.1.2-0.20201008235446-3a8d26a2c1bf h1:JCXrID2CiNOdEo9TN3Dae3EfcYCENv9i/YjXK7fDruo=
-github.com/vmware-tanzu/astrolabe v0.1.2-0.20201008235446-3a8d26a2c1bf/go.mod h1:Af9uI95FSmiaKAiyUFa21rvFAeU195hIv7dMK3XnRag=
+github.com/vmware-tanzu/astrolabe v0.1.2-0.20201114003729-394c01f13c75 h1:Us8oETgxuHdH+PAyzre0p3pWqFNsaO0v5yVpU2TwkT0=
+github.com/vmware-tanzu/astrolabe v0.1.2-0.20201114003729-394c01f13c75/go.mod h1:Af9uI95FSmiaKAiyUFa21rvFAeU195hIv7dMK3XnRag=
 github.com/vmware-tanzu/velero v1.5.1 h1:PMcPfrhv91AfO/NPIWJDVUEql+DUixPnTjg+LTV95yI=
 github.com/vmware-tanzu/velero v1.5.1/go.mod h1:SIyHunlEyLVeKjWR34rv0mLeNVsH5wiR/EmQuUEo1/k=
 github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=

pkg/backupdriver/backup_driver_controller_base.go

@@ -20,6 +20,9 @@ import (
 	"context"
 	"github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/backuprepository"
 	"github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/constants"
+	"github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/utils"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/client-go/informers/core/v1"
 	"strings"
 	"time"
 
@@ -108,11 +111,14 @@ type backupDriverController struct {
 	// Upload queue
 	uploadQueue workqueue.RateLimitingInterface
 
-	// Snapshot queue
+	// Secret queue
+	secretQueue workqueue.RateLimitingInterface
+
+	// DeleteSnapshot queue
 	deleteSnapshotQueue workqueue.RateLimitingInterface
-	// Snapshot Lister
+	// DeleteSnapshot Lister
 	deleteSnapshotLister backupdriverlisters.DeleteSnapshotLister
-	// Snapshot Synced
+	// DeleteSnapshot Synced
 	deleteSnapshotSynced cache.InformerSynced
 
 	// Map supervisor cluster snapshot CRs to guest cluster snapshot CRs
@@ -168,8 +174,10 @@ func NewBackupDriverController(
 	deleteSnapshotQueue := workqueue.NewNamedRateLimitingQueue(rateLimiter, "backup-driver-delete-snapshot-queue")
 	uploadQueue := workqueue.NewNamedRateLimitingQueue(rateLimiter, "backup-driver-upload-queue")
 	svcSnapshotQueue := workqueue.NewNamedRateLimitingQueue(rateLimiter, "backup-driver-svc-snapshot-queue")
+	secretQueue := workqueue.NewNamedRateLimitingQueue(rateLimiter, "backup-driver-secret-queue")
 
 	var svcSnapshotMap map[string]string
+	var secretInformer v1.SecretInformer
 
 	// Configure supervisor cluster queues and caches in the guest
 	// We watch supervisor snapshot CRs for the upload status. If local mode is set, we do not have to watch for upload status
@@ -179,6 +187,10 @@ func NewBackupDriverController(
 
 		cacheSyncs = append(cacheSyncs,
 			svcSnapshotInformer.Informer().HasSynced)
+	} else {
+		// Watch for Secret Changes only in Supervisor/Vanilla setup.
+		secretInformer = informerFactory.Core().V1().Secrets()
+		cacheSyncs = append(cacheSyncs, secretInformer.Informer().HasSynced)
 	}
 
 	ctrl := &backupDriverController{
@@ -204,6 +216,7 @@ func NewBackupDriverController(
 		deleteSnapshotLister:        deleteSnapshotInformer.Lister(),
 		uploadQueue:                 uploadQueue,
 		svcSnapshotQueue:            svcSnapshotQueue,
+		secretQueue:                 secretQueue,
 		cacheSyncs:                  cacheSyncs,
 		svcSnapshotMap:              svcSnapshotMap,
 	}
@@ -275,8 +288,8 @@ func NewBackupDriverController(
 		resyncPeriod,
 	)
 
-	// Configure supervisor cluster informers in the guest
 	if svcKubeConfig != nil {
+		// Configure supervisor cluster informers in the guest
 		svcSnapshotInformer := svcBackupdriverInformerFactory.Backupdriver().V1alpha1().Snapshots()
 		svcSnapshotInformer.Informer().AddEventHandlerWithResyncPeriod(
 			cache.ResourceEventHandlerFuncs{
@@ -286,7 +299,18 @@ func NewBackupDriverController(
 			},
 			resyncPeriod)
 	}
-
+	// Configure secret informer in Supervisor and Vanilla setup only.
+	if secretInformer != nil {
+		secretInformer.Informer().AddEventHandlerWithResyncPeriod(
+			cache.FilteringResourceEventHandler{
+				FilterFunc: utils.GetVcConfigSecretFilterFunc(logger),
+				Handler: cache.ResourceEventHandlerFuncs{
+					AddFunc:    func(obj interface{}) { ctrl.enqueueSecret(obj) },
+					UpdateFunc: func(oldObj, newObj interface{}) { ctrl.enqueueSecret(newObj) },
+				},
+			},
+			constants.DefaultSecretResyncPeriod)
+	}
 	return ctrl
 }
 
@@ -314,6 +338,7 @@ func (ctrl *backupDriverController) Run(
 	defer ctrl.deleteSnapshotQueue.ShutDown()
 	defer ctrl.uploadQueue.ShutDown()
 	defer ctrl.svcSnapshotQueue.ShutDown()
+	defer ctrl.secretQueue.ShutDown()
 
 	ctrl.logger.Infof("Starting backup driver controller")
 	defer ctrl.logger.Infof("Shutting down backup driver controller")
@@ -339,12 +364,71 @@ func (ctrl *backupDriverController) Run(
 
 		if ctrl.svcKubeConfig != nil {
 			go wait.Until(ctrl.svcSnapshotWorker, 0, stopCh)
+		} else {
+			go wait.Until(ctrl.secretWorker, 0, stopCh)
 		}
 	}
 
 	<-stopCh
 }
 
+func (ctrl *backupDriverController) secretWorker() {
+	ctrl.logger.Infof("secretWorker: Enter secretWorker")
+
+	key, quit := ctrl.secretQueue.Get()
+	if quit {
+		return
+	}
+	defer ctrl.secretQueue.Done(key)
+
+	if err := ctrl.syncSecretByKey(key.(string)); err != nil {
+		ctrl.secretQueue.AddRateLimited(key)
+	} else {
+		ctrl.secretQueue.Forget(key)
+	}
+}
+
+func (ctrl *backupDriverController) syncSecretByKey(key string) error {
+	ctrl.logger.Infof("syncSecretByKey: Started Secret processing %s", key)
+	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		ctrl.logger.Errorf("Split meta namespace key of secret %s failed: %v", key, err)
+		return err
+	}
+	// Retrieve the latest Secret.
+	params := make(map[string]interface{})
+	err = utils.RetrieveVcConfigSecret(params, nil, ctrl.logger)
+	if err != nil {
+		ctrl.logger.Errorf("Failed to retrieve the latest vc config secret")
+		return err
+	}
+	ctrl.logger.Infof("Successfully retrieved latest vSphere VC credentials.")
+	err = ctrl.snapManager.ReloadSnapshotManagerIvdPetmConfig(params)
+	if err != nil {
+		ctrl.logger.Errorf("Secret %s/%s Reload failed, err: %v", namespace, name, err)
+		return err
+	}
+	ctrl.logger.Infof("Successfully processed updates in vc configuration.")
+	return nil
+}
+
+func (ctrl *backupDriverController) enqueueSecret(obj interface{}) {
+	// Beware of "xxx deleted" events
+	if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil {
+		obj = unknown.Obj
+	}
+	if secretItem, ok := obj.(*corev1.Secret); ok {
+		ctrl.logger.Infof("enqueueSecret on update: %s", secretItem.Name)
+		objName, err := cache.DeletionHandlingMetaNamespaceKeyFunc(secretItem)
+		if err != nil {
+			ctrl.logger.Errorf("failed to get key from object: %v, %v", err, secretItem)
+			return
+		}
+		ctrl.logger.Infof("enqueueSecret: enqueued %q for sync", objName)
+		ctrl.secretQueue.Add(objName)
+	}
+}
+
 // snapshotWorker is the main worker for snapshot request.
 func (ctrl *backupDriverController) snapshotWorker() {
 	ctrl.logger.Infof("snapshotWorker: Enter snapshotWorker")

pkg/cmd/backupdriver/cli/server/server.go

@@ -19,6 +19,7 @@ package server
 import (
 	"context"
 	"fmt"
+	server2 "github.com/vmware-tanzu/astrolabe/pkg/server"
 	"github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/constants"
 	"log"
 	"net/http"
@@ -30,7 +31,6 @@ import (
 	"time"
 
 	"github.com/vmware-tanzu/astrolabe/pkg/astrolabe"
-	server2 "github.com/vmware-tanzu/astrolabe/pkg/server"
 	"k8s.io/client-go/util/workqueue"
 
 	"github.com/pkg/errors"
@@ -82,8 +82,8 @@ func NewCommand(f client.Factory) *cobra.Command {
 			kubeConfig:         "",
 			resyncPeriod:       constants.ResyncPeriod,
 			workers:            cmd.DefaultBackupWorkers,
-			retryIntervalStart: cmd.DefaultRetryIntervalStart,
-			retryIntervalMax:   cmd.DefaultRetryIntervalMax,
+			retryIntervalStart: constants.DefaultRetryIntervalStart,
+			retryIntervalMax:   constants.DefaultRetryIntervalMax,
 		}
 	)
 

pkg/cmd/constants.go

@@ -16,8 +16,6 @@ limitations under the License.
 
 package cmd
 
-import "time"
-
 const (
 	// the port where prometheus metrics are exposed
 	DefaultMetricsAddress = ":8085"
@@ -30,7 +28,5 @@ const (
 	DefaultInsecureFlag       bool = true
 	DefaultVCConfigFromSecret bool = true
 
-	DefaultBackupWorkers      = 1
-	DefaultRetryIntervalStart = time.Second
-	DefaultRetryIntervalMax   = 5 * time.Minute
+	DefaultBackupWorkers = 1
 )

pkg/cmd/datamgr/cli/server/server.go

@@ -19,6 +19,8 @@ package server
 import (
 	"context"
 	"fmt"
+	"github.com/vmware-tanzu/astrolabe/pkg/common/vsphere"
+	server2 "github.com/vmware-tanzu/astrolabe/pkg/server"
 	"github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/constants"
 	"log"
 	"net/http"
@@ -30,8 +32,6 @@ import (
 	"time"
 
 	"github.com/vmware-tanzu/astrolabe/pkg/astrolabe"
-	"github.com/vmware-tanzu/astrolabe/pkg/ivd"
-	astrolabeServer "github.com/vmware-tanzu/astrolabe/pkg/server"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -156,6 +156,7 @@ type server struct {
 	dataMover             *dataMover.DataMover
 	snapManager           *snapshotmgr.SnapshotManager
 	externalDataMgr       bool
+	vcConfigSecret        bool
 }
 
 func (s *server) run() error {
@@ -188,27 +189,27 @@ func getVCConfigParams(config serverConfig, params map[string]interface{}, logge
 	if config.vCenter == "" {
 		return errors.New("getVCConfigParams: parameter vcenter-address not provided")
 	}
-	params[ivd.HostVcParamKey] = config.vCenter
+	params[vsphere.HostVcParamKey] = config.vCenter
 
 	if config.user == "" {
 		return errors.New("getVCConfigParams: parameter vcenter-user not provided")
 	}
-	params[ivd.UserVcParamKey] = config.user
+	params[vsphere.UserVcParamKey] = config.user
 
 	passwd := os.Getenv("VC_PASSWORD")
 	if passwd == "" {
 		logger.Warnf("getVCConfigParams: Environment variable VC_PASSWORD not set or empty")
 	}
-	params[ivd.PasswordVcParamKey] = passwd
+	params[vsphere.PasswordVcParamKey] = passwd
 
 	if config.clusterId == "" {
 		return errors.New("getVCConfigParams: parameter vcenter-user not provided")
 	}
-	params[ivd.ClusterVcParamKey] = config.clusterId
+	params[vsphere.ClusterVcParamKey] = config.clusterId
 
 	// Below vc configuration params are optional
-	params[ivd.PortVcParamKey] = config.port
-	params[ivd.InsecureFlagVcParamKey] = strconv.FormatBool(config.insecureFlag)
+	params[vsphere.PortVcParamKey] = config.port
+	params[vsphere.InsecureFlagVcParamKey] = strconv.FormatBool(config.insecureFlag)
 
 	return nil
 }
@@ -244,7 +245,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 			return nil, err
 		}
 
-		logger.Infof("VC configuration provided by user for :%s", ivdParams[ivd.HostVcParamKey])
+		logger.Infof("VC configuration provided by user for :%s", ivdParams[vsphere.HostVcParamKey])
 	}
 
 	snapshotMgrConfig := make(map[string]string)
@@ -259,7 +260,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 	}
 	s3RepoParams := make(map[string]interface{})
 
-	configInfo := astrolabeServer.NewConfigInfo(peConfigs, s3Config)
+	configInfo := server2.NewConfigInfo(peConfigs, s3Config)
 	snapshotMgr, err := snapshotmgr.NewSnapshotManagerFromConfig(configInfo, s3RepoParams, snapshotMgrConfig,
 		clientConfig, logger)
 	if err != nil {
@@ -296,6 +297,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s
 		dataMover:             clusterDataMover,
 		snapManager:           snapshotMgr,
 		externalDataMgr:       externalDataMgr,
+		vcConfigSecret:        config.vcConfigFromSecret,
 	}
 	return s, nil
 }
@@ -354,6 +356,21 @@ func (s *server) runControllers() error {
 		os.Getenv("NODE_NAME"),
 	)
 
+	if !s.externalDataMgr && s.vcConfigSecret {
+		s.logger.Infof("Watching for vc config secret changes.")
+		vcConfigController := controller.NewVcConfigController(
+			s.logger,
+			s.kubeInformerFactory.Core().V1().Secrets(),
+			s.dataMover,
+			s.snapManager,
+		)
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			vcConfigController.Run(s.ctx, 1)
+		}()
+	}
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()

pkg/constants/constants.go

@@ -37,6 +37,8 @@ const (
 const (
 	// Duration after which Reflector resyncs CRs and calls UpdateFunc on each of the existing CRs.
 	ResyncPeriod = 30 * time.Second
+
+	DefaultSecretResyncPeriod = 5 * time.Minute
 )
 
 // configuration constants for the volume snapshot plugin
@@ -135,8 +137,8 @@ const (
 
 // feature flog constants
 const (
-	VSphereLocalModeFlag        = "local-mode"
-	VSphereLocalModeFeature     = "EnableLocalMode"
+	VSphereLocalModeFlag    = "local-mode"
+	VSphereLocalModeFeature = "EnableLocalMode"
 )
 
 const (
@@ -308,3 +310,8 @@ const (
 	ImageContainerComponent = "Container"
 	ImageVersionComponent = "Version"
 )
+
+const (
+	DefaultRetryIntervalStart = time.Second
+	DefaultRetryIntervalMax   = 5 * time.Minute
+)