Skip to content

Commit

Permalink
Add bucket client support for gcs, azure; update config flag validati…
Browse files Browse the repository at this point in the history 
…on; update tests; go mod vendor & go mod tidy

Signed-off-by: JordanRushing <[email protected]>

Update mockBucket instantiation in test; deny admission requests when failing to bootstrap the s3 bucket client

Signed-off-by: JordanRushing <[email protected]>

Clean up s3 config struct

Signed-off-by: JordanRushing <[email protected]>

Change s3 config creation semantics for zoneTracker

Signed-off-by: JordanRushing <[email protected]>

Add support for gcs and azure object storage clients; go mod vendor & go mod tidy

Signed-off-by: JordanRushing <[email protected]>
  • Loading branch information
@JordanRushing
JordanRushing committed Dec 4, 2023
1 parent b11ec77 commit 33dd5e5
Show file tree
Hide file tree
Showing 833 changed files with 230,090 additions and 188 deletions.
12 changes: 11 additions & 1 deletion cmd/rollout-operator/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ type config struct {
bucketName string
objectStorageEndpoint string
objectStorageRegion string
accountName string
}

func (cfg *config) register(fs *flag.FlagSet) {
Expand Down Expand Up @@ -91,6 +92,7 @@ func (cfg *config) register(fs *flag.FlagSet) {
fs.StringVar(&cfg.bucketName, "bucket-name", "", "The name of the bucket to use for the zone tracker")
fs.StringVar(&cfg.objectStorageEndpoint, "endpoint", "", "The endpoint to use for the object storage provider")
fs.StringVar(&cfg.objectStorageRegion, "region", "", "The region to use for the object storage provider")
fs.StringVar(&cfg.accountName, "account-name", "", "The account name to use for the Azure object storage provider")
}

func (cfg config) validate() error {
Expand Down Expand Up @@ -119,6 +121,14 @@ func (cfg config) validate() error {
if !supported {
return fmt.Errorf("the provided object storage provider is not supported (currently supported providers: %s)", strings.Join(supportedProviders, ", "))
}
if cfg.objectStorageProvider == "azure" && cfg.accountName == "" {
return errors.New("the account name for the Azure object storage provider has not been specified")
}
if cfg.objectStorageProvider == "s3" {
if cfg.objectStorageEndpoint == "" || cfg.objectStorageRegion == "" {
return errors.New("the endpoint and region for the S3 object storage provider have not been specified")
}
}
}

return nil
Expand Down Expand Up @@ -220,7 +230,7 @@ func maybeStartTLSServer(cfg config, logger log.Logger, kubeClient *kubernetes.C
}

prepDownscaleAdmitFunc := func(ctx context.Context, logger log.Logger, ar v1.AdmissionReview, api *kubernetes.Clientset) *v1.AdmissionResponse {
return admission.PrepareDownscale(ctx, logger, ar, api, cfg.useZoneTracker, cfg.objectStorageProvider, cfg.bucketName, cfg.objectStorageEndpoint, cfg.objectStorageRegion)
return admission.PrepareDownscale(ctx, logger, ar, api, cfg.useZoneTracker, cfg.objectStorageProvider, cfg.bucketName, cfg.objectStorageEndpoint, cfg.objectStorageRegion, cfg.accountName)
}

tlsSrv, err := newTLSServer(cfg.serverTLSPort, logger, cert)
Expand Down
26 changes: 25 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,16 @@ require (
)

require (
cloud.google.com/go v0.110.10 // indirect
cloud.google.com/go/compute v1.23.3 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.5 // indirect
cloud.google.com/go/storage v1.35.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.16.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.1 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.11.0 // indirect
Expand All @@ -35,11 +45,25 @@ require (
github.com/aws/smithy-go v1.11.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 // indirect
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.61 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/rs/xid v1.5.0 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/api v0.150.0 // indirect
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/grpc v1.59.0 // indirect
)

require (
Expand Down Expand Up @@ -75,7 +99,7 @@ require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/goodhosts/hostsfile v0.1.1 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-containerregistry v0.16.1 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.4.0 // indirect
Expand Down
12 changes: 10 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/cli v24.0.5+incompatible h1:WeBimjvS0eKdH4Ygx+ihVq1Q++xg36M/rMi4aXAvodc=
github.com/docker/cli v24.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
Expand Down Expand Up @@ -276,21 +278,25 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ=
github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
Expand Down Expand Up @@ -665,6 +671,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
Expand Down Expand Up @@ -737,6 +744,7 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
14 changes: 5 additions & 9 deletions pkg/admission/prep_downscale.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"io"
"net/http"
"os"
"time"

"github.com/go-kit/log"
Expand All @@ -30,20 +29,17 @@ const (
PrepareDownscaleWebhookPath = "/admission/prepare-downscale"
)

func PrepareDownscale(ctx context.Context, logger log.Logger, ar v1.AdmissionReview, api *kubernetes.Clientset, useZoneTracker bool, objectStorageProvider string, bucketName string, endpoint string, region string) *v1.AdmissionResponse {
func PrepareDownscale(ctx context.Context, logger log.Logger, ar v1.AdmissionReview, api *kubernetes.Clientset, useZoneTracker bool, objectStorageProvider string, bucketName string, endpoint string, region string, accountName string) *v1.AdmissionResponse {
client := &http.Client{
Timeout: 5 * time.Second,
}

var zt *zoneTracker
if useZoneTracker {
// TODO(jordanrushing): fix bucket creation semantics and wire-in config supporting multiple CSPs
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
// TODO(jordanrushing): stop hardcoding insecure: true
s3Config, _ := config.CreateS3ConfigYaml(bucketName, endpoint, region, accessKey, secretKey, true)
bkt, _ := newS3BucketClient(s3Config, logger)

bkt, err := newBucketClient(ctx, objectStorageProvider, bucketName, endpoint, region, accountName, logger)
if err != nil {
return deny("downscale not allowed due to error while creating object storage bucket client with zoneTracker enabled: %v", err)
}
zt = newZoneTracker(bkt, config.ZoneTrackerKey)
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/admission/prep_downscale_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/go-kit/log/level"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/thanos-io/objstore"
v1 "k8s.io/api/admission/v1"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -537,7 +538,7 @@ func testPrepDownscaleWebhookWithZoneTracker(t *testing.T, oldReplicas, newRepli
api := fake.NewSimpleClientset(objects...)
f := &fakeHttpClient{statusCode: params.statusCode}

bkt := &mockBucket{data: make(map[string][]byte)}
bkt := &mockBucket{bkt: objstore.NewInMemBucket(), data: make(map[string][]byte)}
zt := newZoneTracker(bkt, "testkey")
admissionResponse := prepareDownscale(ctx, logger, ar, api, f, zt)
require.Equal(t, params.allowed, admissionResponse.Allowed, "Unexpected result for allowed: got %v, expected %v", admissionResponse.Allowed, params.allowed)
Expand Down
85 changes: 84 additions & 1 deletion pkg/admission/zone_tracker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,17 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"sync"
"time"

"github.com/go-kit/log"
"github.com/grafana/rollout-operator/pkg/config"
"github.com/thanos-io/objstore"
"github.com/thanos-io/objstore/providers/azure"
"github.com/thanos-io/objstore/providers/gcs"
"github.com/thanos-io/objstore/providers/s3"
"gopkg.in/yaml.v2"
appsv1 "k8s.io/api/apps/v1"
)

Expand Down Expand Up @@ -142,7 +146,86 @@ func newZoneTracker(bkt objstore.Bucket, key string) *zoneTracker {
}
}

// TODO(jordanrushing): Make this generic for supported providers
// newBucketClient creates a new object storage bucket client based on the minimum provided configuration flags and provider
// Supported providers are s3, azure, and gcs
func newBucketClient(ctx context.Context, provider string, bucket string, endpoint string, region string, accountName string, logger log.Logger) (objstore.Bucket, error) {
switch provider {
case "s3":
config, err := createS3ConfigYaml(bucket, endpoint, region)
if err != nil {
return nil, err
}
return newS3BucketClient(config, logger)
case "azure":
config, err := createAzureConfigYaml(bucket, accountName)
if err != nil {
return nil, err
}
return newAzureBucketClient(config, logger)
case "gcs":
config, err := createGCSConfigYaml(ctx, bucket)
if err != nil {
return nil, err
}
return newGCSBucketClient(ctx, config, logger)
default:
return nil, fmt.Errorf("unable to provision an object storage bucket client based on the provided configuration flags")
}
}

func newS3BucketClient(config []byte, logger log.Logger) (objstore.Bucket, error) {
return s3.NewBucket(logger, config, "grafana-rollout-operator")
}

func newAzureBucketClient(config []byte, logger log.Logger) (objstore.Bucket, error) {
return azure.NewBucket(logger, config, "grafana-rollout-operator")
}

func newGCSBucketClient(ctx context.Context, config []byte, logger log.Logger) (objstore.Bucket, error) {
return gcs.NewBucket(ctx, logger, config, "grafana-rollout-operator")
}

func createS3ConfigYaml(bucket, endpoint, region string) ([]byte, error) {
cfg := &s3.Config{
Bucket: bucket,
Endpoint: endpoint,
Region: region,
AccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
SecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
}

data, err := yaml.Marshal(cfg)
if err != nil {
return nil, err
}

return data, nil
}

func createAzureConfigYaml(container, accountName string) ([]byte, error) {
cfg := &azure.Config{
ContainerName: container,
StorageAccountKey: os.Getenv("AZURE_BLOB_SECRET_ACCESS_KEY"),
StorageAccountName: accountName,
}

data, err := yaml.Marshal(cfg)
if err != nil {
return nil, err
}

return data, nil
}

func createGCSConfigYaml(ctx context.Context, bucket string) ([]byte, error) {
cfg := &gcs.Config{
Bucket: bucket,
}

data, err := yaml.Marshal(cfg)
if err != nil {
return nil, err
}

return data, nil
}
35 changes: 0 additions & 35 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
package config

import (
"gopkg.in/yaml.v2"
)

const (
// NoDownscaleLabelKey is the label to prevent downscaling of a statefulset
NoDownscaleLabelKey = "grafana.com/no-downscale"
Expand Down Expand Up @@ -36,34 +32,3 @@ const (
// ZoneTrackerKey is the key used to store the zone tracker data in object storage
ZoneTrackerKey = "zone-tracker/last-downscaled.json"
)

type S3Config struct {
Type string `yaml:"type"`
Config struct {
Bucket string `yaml:"bucket"`
Endpoint string `yaml:"endpoint"`
Region string `yaml:"region"`
AccessKey string `yaml:"access_key"`
SecretKey string `yaml:"secret_key"`
Insecure bool `yaml:"insecure"`
} `yaml:"config"`
}

func CreateS3ConfigYaml(s3Bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey string, insecure bool) ([]byte, error) {
cfg := S3Config{
Type: "S3",
}
cfg.Config.Bucket = s3Bucket
cfg.Config.Endpoint = s3Endpoint
cfg.Config.Region = s3Region
cfg.Config.AccessKey = s3AccessKey
cfg.Config.SecretKey = s3SecretKey
cfg.Config.Insecure = true

data, err := yaml.Marshal(cfg)
if err != nil {
return nil, err
}

return data, nil
}
Loading

0 comments on commit 33dd5e5

Please sign in to comment.