From c66bc0eaf70811573c19807be9365ddfba2e8719 Mon Sep 17 00:00:00 2001 From: xinyanw409 Date: Tue, 15 Nov 2022 16:45:13 -0800 Subject: [PATCH 1/4] Spin datamgr to datamgr_install and datamgr_server --- Dockerfile-backup-driver | 1 - Dockerfile-plugin | 1 - Makefile | 67 +++++++++++++++++-- cmd/data-manager-for-plugin-server/main.go | 36 ++++++++++ cmd/data-manager-for-plugin/main.go | 2 +- ...ate-deployment-for-backupdriver-guest.yaml | 2 - .../create-deployment-for-backupdriver.yaml | 2 - .../cli/install/install.go | 0 .../datamgr_install.go} | 5 +- .../cli/server/server.go | 0 pkg/cmd/datamgr_server/datamgr_server.go | 61 +++++++++++++++++ pkg/install/deployment.go | 4 -- scripts/install.sh | 2 +- 13 files changed, 161 insertions(+), 22 deletions(-) create mode 100644 cmd/data-manager-for-plugin-server/main.go rename pkg/cmd/{datamgr => datamgr_install}/cli/install/install.go (100%) rename pkg/cmd/{datamgr/datamgr.go => datamgr_install/datamgr_install.go} (93%) rename pkg/cmd/{datamgr => datamgr_server}/cli/server/server.go (100%) create mode 100644 pkg/cmd/datamgr_server/datamgr_server.go diff --git a/Dockerfile-backup-driver b/Dockerfile-backup-driver index da9b5ad0..cbedf942 100644 --- a/Dockerfile-backup-driver +++ b/Dockerfile-backup-driver @@ -13,6 +13,5 @@ # limitations under the License. FROM photon:4.0 -COPY /bin/linux/amd64/lib/vmware-vix-disklib/lib64/* /vddkLibs/ ADD /bin/linux/amd64/backup-driver* /backup-driver ENTRYPOINT ["/backup-driver"] diff --git a/Dockerfile-plugin b/Dockerfile-plugin index 2f8d5da0..736c19cb 100644 --- a/Dockerfile-plugin +++ b/Dockerfile-plugin @@ -19,7 +19,6 @@ ADD /bin/linux/amd64/velero-* /plugins/ ADD /bin/linux/amd64/data-* / ADD /bin/linux/amd64/backup-driver* / COPY /bin/linux/amd64/install.sh /scripts/ -COPY /bin/linux/amd64/lib/vmware-vix-disklib/lib64/* /plugins/ ENV LD_LIBRARY_PATH=/plugins COPY --from=busybox /bin/sh /bin/sh COPY --from=busybox /bin/chmod /bin/chmod diff --git a/Makefile b/Makefile index 2893e880..0e529596 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,8 @@ VDDK_LIBS:= $(LIB_DIR)/vmware-vix-disklib-distrib/lib64 # The binary to build (just the basename). PLUGIN_BIN ?= velero-plugin-for-vsphere DATAMGR_BIN ?= data-manager-for-plugin +DATAMGR_INSTALL_BIN ?= data-manager-for-plugin-install +DATAMGR_SERVER_BIN ?= data-manager-for-plugin-server BACKUPDRIVER_BIN ?= backup-driver VSPHERE_ASTROLABE ?= vsphere-astrolabe @@ -70,7 +72,7 @@ ifeq (,$(wildcard $(GOPATH)/src/$(VDDK_LIBS))) $(error "$(GOPATH)/src/$(VDDK_LIBS) cannot find vddk libs in path. Please refer to: https://github.com/vmware/virtual-disks#dependency, download the VDDK tarball to the directory $(GOPATH)/src/$(LIB_DIR)/ and untar it") endif -plugin: datamgr backup-driver +plugin: datamgr-install backup-driver @echo "making: $@" $(MAKE) build BIN=$(PLUGIN_BIN) VERSION=$(VERSION) @@ -78,6 +80,14 @@ datamgr: @echo "making: $@" $(MAKE) build BIN=$(DATAMGR_BIN) VERSION=$(VERSION) +datamgr-install: + @echo "making: $@" + $(MAKE) build BIN=$(DATAMGR_INSTALL_BIN) VERSION=$(VERSION) + +datamgr-server: + @echo "making: $@" + $(MAKE) build BIN=$(DATAMGR_SERVER_BIN) VERSION=$(VERSION) + backup-driver: @echo "making: $@" $(MAKE) build BIN=$(BACKUPDRIVER_BIN) VERSION=$(VERSION) @@ -119,6 +129,44 @@ _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs copy-astrolabe TTY := $(shell tty -s && echo "-t") shell: build-dirs + @echo "running docker: $@" + docker run \ + -e GOFLAGS \ + -i $(TTY) \ + --rm \ + -u $$(id -u):$$(id -g) \ + -v $$(pwd)/.go/pkg:/go/pkg:delegated \ + -v $$(pwd)/.go/src:/go/src:delegated \ + -v $$(pwd)/.go/std:/go/std:delegated \ + -v $$(pwd):/go/src/$(PKG):delegated \ + -v "$$(pwd)/_output/bin:/output:delegated" \ + -v $$(pwd)/.go/std/$(GOOS)_$(GOARCH):/usr/local/go/pkg/$(GOOS)_$(GOARCH)_static:delegated \ + -v "$$(pwd)/.go/go-build:/.cache/go-build:delegated" \ + -e CGO_ENABLED=1 \ + -e GOPATH=/go \ + -w /go/src/$(PKG) \ + $(BUILDER_IMAGE) \ + /bin/sh $(CMD) + +build-datamgr-server: _output/bin/$(GOOS)/$(GOARCH)/$(DATAMGR_SERVER_BIN)/ + +_output/bin/$(GOOS)/$(GOARCH)/$(DATAMGR_SERVER_BIN): build-dirs copy-astrolabe + @echo "building: $@" + $(MAKE) shell-datamgr-server CMD="-c '\ + GOOS=$(GOOS) \ + GOARCH=$(GOARCH) \ + REGISTRY=$(REGISTRY) \ + VERSION=$(VERSION) \ + PKG=$(PKG) \ + BIN=$(BIN) \ + GIT_SHA=$(GIT_SHA) \ + GIT_DIRTY=\"$(GIT_DIRTY)\" \ + OUTPUT_DIR=/output/$(GOOS)/$(GOARCH) \ + GO111MODULE=on \ + GOFLAGS=-mod=readonly \ + ./hack/build.sh'" + +shell-datamgr-server: build-dirs @echo "running docker: $@" docker run \ -e GOFLAGS \ @@ -166,20 +214,27 @@ copy-vix-libs: copy-install-script: cp $$(pwd)/scripts/install.sh _output/bin/$(GOOS)/$(GOARCH) -build-container: copy-vix-libs container-name +build-container: container-name + cp $(DOCKERFILE) _output/bin/$(GOOS)/$(GOARCH)/$(DOCKERFILE) + docker build -t $(IMAGE):$(VERSION) -f _output/bin/$(GOOS)/$(GOARCH)/$(DOCKERFILE) _output + +build-datamgr-server-container: copy-vix-libs container-name cp $(DOCKERFILE) _output/bin/$(GOOS)/$(GOARCH)/$(DOCKERFILE) docker build -t $(IMAGE):$(VERSION) -f _output/bin/$(GOOS)/$(GOARCH)/$(DOCKERFILE) _output plugin-container: all copy-install-script $(MAKE) build-container IMAGE=$(PLUGIN_IMAGE) DOCKERFILE=$(PLUGIN_DOCKERFILE) VERSION=$(VERSION) -datamgr-container: datamgr - $(MAKE) build-container BIN=$(DATAMGR_BIN) IMAGE=$(DATAMGR_IMAGE) DOCKERFILE=$(DATAMGR_DOCKERFILE) VERSION=$(VERSION) +datamgr-install-container: datamgr-install + $(MAKE) build-container BIN=$(DATAMGR_BIN_INSTALL) IMAGE=$(DATAMGR_IMAGE) DOCKERFILE=$(DATAMGR_DOCKERFILE) VERSION=$(VERSION) + +datamgr-server-container: datamgr-server + $(MAKE) build-container BIN=$(DATAMGR_BIN_SERVER) IMAGE=$(DATAMGR_IMAGE) DOCKERFILE=$(DATAMGR_DOCKERFILE) VERSION=$(VERSION) backup-driver-container: backup-driver $(MAKE) build-container BIN=$(BACKUPDRIVER_BIN) IMAGE=$(BACKUPDRIVER_IMAGE) DOCKERFILE=$(BACKUPDRIVER_DOCKERFILE) VERSION=$(VERSION) -container: plugin-container datamgr-container backup-driver-container +container: plugin-container datamgr-install-container datamgr-server-container backup-driver-container update: @echo "updating CRDs" @@ -188,7 +243,7 @@ update: push-plugin: plugin-container docker push $(PLUGIN_IMAGE):$(VERSION) -push-datamgr: datamgr-container +push-datamgr: datamgr-install-container datamgr-server-container docker push $(DATAMGR_IMAGE):$(VERSION) push-backup-driver: backup-driver-container diff --git a/cmd/data-manager-for-plugin-server/main.go b/cmd/data-manager-for-plugin-server/main.go new file mode 100644 index 00000000..a36bc38d --- /dev/null +++ b/cmd/data-manager-for-plugin-server/main.go @@ -0,0 +1,36 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "path/filepath" + + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/cmd" + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/cmd/datamgr_server" + + "k8s.io/klog" +) + +func main() { + defer klog.Flush() + + baseName := filepath.Base(os.Args[0]) + + err := datamgr.NewCommand(baseName).Execute() + cmd.CheckError(err) +} diff --git a/cmd/data-manager-for-plugin/main.go b/cmd/data-manager-for-plugin/main.go index d4e10fe8..995e2e76 100644 --- a/cmd/data-manager-for-plugin/main.go +++ b/cmd/data-manager-for-plugin/main.go @@ -21,7 +21,7 @@ import ( "path/filepath" "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/cmd" - "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/cmd/datamgr" + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/cmd/datamgr_install" "k8s.io/klog" ) diff --git a/deployment/create-deployment-for-backupdriver-guest.yaml b/deployment/create-deployment-for-backupdriver-guest.yaml index fbbe5ef0..8571a21a 100644 --- a/deployment/create-deployment-for-backupdriver-guest.yaml +++ b/deployment/create-deployment-for-backupdriver-guest.yaml @@ -50,8 +50,6 @@ spec: value: velero - name: VELERO_SCRATCH_DIR value: /scratch - - name: LD_LIBRARY_PATH - value: /vddkLibs image: dpcpinternal/backup-driver: imagePullPolicy: IfNotPresent name: backup-driver diff --git a/deployment/create-deployment-for-backupdriver.yaml b/deployment/create-deployment-for-backupdriver.yaml index 982fb9f0..ce5958bb 100644 --- a/deployment/create-deployment-for-backupdriver.yaml +++ b/deployment/create-deployment-for-backupdriver.yaml @@ -50,8 +50,6 @@ spec: value: velero - name: VELERO_SCRATCH_DIR value: /scratch - - name: LD_LIBRARY_PATH - value: /vddkLibs image: dpcpinternal/backup-driver: imagePullPolicy: IfNotPresent name: backup-driver diff --git a/pkg/cmd/datamgr/cli/install/install.go b/pkg/cmd/datamgr_install/cli/install/install.go similarity index 100% rename from pkg/cmd/datamgr/cli/install/install.go rename to pkg/cmd/datamgr_install/cli/install/install.go diff --git a/pkg/cmd/datamgr/datamgr.go b/pkg/cmd/datamgr_install/datamgr_install.go similarity index 93% rename from pkg/cmd/datamgr/datamgr.go rename to pkg/cmd/datamgr_install/datamgr_install.go index a0eb4e61..79741ab6 100644 --- a/pkg/cmd/datamgr/datamgr.go +++ b/pkg/cmd/datamgr_install/datamgr_install.go @@ -21,10 +21,8 @@ import ( "fmt" "os" - "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/cmd/datamgr/cli/install" - "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/cmd/datamgr/cli/server" - "github.com/spf13/cobra" + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/cmd/datamgr_install/cli/install" "k8s.io/klog" "github.com/vmware-tanzu/velero/pkg/client" @@ -51,7 +49,6 @@ func NewCommand(name string) *cobra.Command { f.BindFlags(c.PersistentFlags()) c.AddCommand( - server.NewCommand(f), install.NewCommand(f), ) diff --git a/pkg/cmd/datamgr/cli/server/server.go b/pkg/cmd/datamgr_server/cli/server/server.go similarity index 100% rename from pkg/cmd/datamgr/cli/server/server.go rename to pkg/cmd/datamgr_server/cli/server/server.go diff --git a/pkg/cmd/datamgr_server/datamgr_server.go b/pkg/cmd/datamgr_server/datamgr_server.go new file mode 100644 index 00000000..7dda8241 --- /dev/null +++ b/pkg/cmd/datamgr_server/datamgr_server.go @@ -0,0 +1,61 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datamgr + +import ( + "flag" + "fmt" + "os" + + "github.com/vmware-tanzu/velero-plugin-for-vsphere/pkg/cmd/datamgr_server/cli/server" + + "github.com/spf13/cobra" + "k8s.io/klog" + + "github.com/vmware-tanzu/velero/pkg/client" + //"github.com/vmware-tanzu/velero/pkg/features" +) + +func NewCommand(name string) *cobra.Command { + // Load the config here so that we can extract features from it. + config, err := client.LoadConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "WARNING: Error reading config file: %v\n", err) + } + + c := &cobra.Command{ + Use: name, + Short: "Upload and download snapshots of persistent volume on vSphere kubernetes cluster", + Long: `Data manager is a component in Velero vSphere plugin for + moving local snapshotted data from/to remote durable persistent storage. + Specifically, the data manager component is supposed to be running + in separate container from the velero server`, + } + + f := client.NewFactory(name, config) + f.BindFlags(c.PersistentFlags()) + + c.AddCommand( + server.NewCommand(f), + ) + + // init and add the klog flags + klog.InitFlags(flag.CommandLine) + c.PersistentFlags().AddGoFlagSet(flag.CommandLine) + + return c +} diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index b2280082..1332d9d7 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -131,10 +131,6 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment Name: "VELERO_SCRATCH_DIR", Value: "/scratch", }, - { - Name: "LD_LIBRARY_PATH", - Value: "/vddkLibs", - }, }, Resources: c.resources, }, diff --git a/scripts/install.sh b/scripts/install.sh index ed1898f6..544ce369 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -16,4 +16,4 @@ cp /plugins/* /target/. /backup-driver install -/data-manager-for-plugin install +/data-manager-for-plugin-install install From 91a31f3017d916be5421512488d5525d481b0e38 Mon Sep 17 00:00:00 2001 From: xinyanw409 Date: Thu, 17 Nov 2022 14:11:16 -0800 Subject: [PATCH 2/4] Change package name --- pkg/cmd/datamgr_install/datamgr_install.go | 2 +- pkg/cmd/datamgr_server/datamgr_server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/datamgr_install/datamgr_install.go b/pkg/cmd/datamgr_install/datamgr_install.go index 79741ab6..257d5e6b 100644 --- a/pkg/cmd/datamgr_install/datamgr_install.go +++ b/pkg/cmd/datamgr_install/datamgr_install.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package datamgr +package datamgr_install import ( "flag" diff --git a/pkg/cmd/datamgr_server/datamgr_server.go b/pkg/cmd/datamgr_server/datamgr_server.go index 7dda8241..901b8969 100644 --- a/pkg/cmd/datamgr_server/datamgr_server.go +++ b/pkg/cmd/datamgr_server/datamgr_server.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package datamgr +package datamgr_server import ( "flag" From 4ee8c08cc08a2f433b59f3d73dedcd3b9a736869 Mon Sep 17 00:00:00 2001 From: xinyanw409 Date: Thu, 17 Nov 2022 14:34:53 -0800 Subject: [PATCH 3/4] rename pkg name --- .../main.go | 2 +- cmd/data-manager-for-plugin-server/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename cmd/{data-manager-for-plugin => data-manager-for-plugin-install}/main.go (94%) diff --git a/cmd/data-manager-for-plugin/main.go b/cmd/data-manager-for-plugin-install/main.go similarity index 94% rename from cmd/data-manager-for-plugin/main.go rename to cmd/data-manager-for-plugin-install/main.go index 995e2e76..414526f1 100644 --- a/cmd/data-manager-for-plugin/main.go +++ b/cmd/data-manager-for-plugin-install/main.go @@ -31,6 +31,6 @@ func main() { baseName := filepath.Base(os.Args[0]) - err := datamgr.NewCommand(baseName).Execute() + err := datamgr_install.NewCommand(baseName).Execute() cmd.CheckError(err) } diff --git a/cmd/data-manager-for-plugin-server/main.go b/cmd/data-manager-for-plugin-server/main.go index a36bc38d..b04e876a 100644 --- a/cmd/data-manager-for-plugin-server/main.go +++ b/cmd/data-manager-for-plugin-server/main.go @@ -31,6 +31,6 @@ func main() { baseName := filepath.Base(os.Args[0]) - err := datamgr.NewCommand(baseName).Execute() + err := datamgr_server.NewCommand(baseName).Execute() cmd.CheckError(err) } From 6594d657739fec8ea2ea150f1ba92cb2a681f791 Mon Sep 17 00:00:00 2001 From: xinyanw409 Date: Thu, 8 Dec 2022 10:56:26 -0800 Subject: [PATCH 4/4] Fix compile issue --- Dockerfile-datamgr | 5 +++-- Makefile | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile-datamgr b/Dockerfile-datamgr index c35bae3b..6f04034d 100644 --- a/Dockerfile-datamgr +++ b/Dockerfile-datamgr @@ -15,10 +15,11 @@ FROM busybox:1.33.1 AS busybox FROM photon:4.0 -ADD /bin/linux/amd64/data-* /datamgr +ADD /bin/linux/amd64/data-manager-for-plugin-install /datamgr-install +ADD /bin/linux/amd64/data-manager-for-plugin /datamgr-server COPY /bin/linux/amd64/lib/vmware-vix-disklib/lib64/* /vddkLibs/ COPY --from=busybox /bin/sh /bin/sh COPY --from=busybox /bin/cp /bin/cp COPY --from=busybox /bin/tar /bin/tar COPY --from=busybox /bin/ls /bin/ls -ENTRYPOINT ["/datamgr"] +ENTRYPOINT ["/datamgr-install & /datamgr-server"] diff --git a/Makefile b/Makefile index 0e529596..5a135d12 100644 --- a/Makefile +++ b/Makefile @@ -135,6 +135,7 @@ shell: build-dirs -i $(TTY) \ --rm \ -u $$(id -u):$$(id -g) \ + -v $$(pwd)/.libs/vmware-vix-disklib-distrib:/usr/local/vmware-vix-disklib-distrib:delegated \ -v $$(pwd)/.go/pkg:/go/pkg:delegated \ -v $$(pwd)/.go/src:/go/src:delegated \ -v $$(pwd)/.go/std:/go/std:delegated \