Skip to content

Commit

Permalink
Adds the ability to override backup_yaml values using a capitalized s…
Browse files Browse the repository at this point in the history
…nake case format (--set MILVUS_USER=Marco)"

Command to print the current backup config file in yaml format to stdio

Signed-off-by: romel.campbell <[email protected]>
  • Loading branch information
roca committed Sep 23, 2023
1 parent b643d2d commit f7fd49b
Show file tree
Hide file tree
Showing 9 changed files with 151 additions and 17 deletions.
3 changes: 3 additions & 0 deletions .goreleaser.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ builds:
goos:
- linux
- darwin
goarch:
- amd64
- arm64
ldflags:
- -w -s
- -X main.version={{ .Version }}
Expand Down
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
Romel
# Milvus-backup

Milvus-Backup is a tool that allows users to backup and restore Milvus data. This tool can be utilized either through the command line or an API server.
Expand Down
102 changes: 102 additions & 0 deletions cmd/backup_yaml.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package cmd

import (
"fmt"
"strings"
"github.com/spf13/cobra"
"github.com/zilliztech/milvus-backup/core/paramtable"
"gopkg.in/yaml.v3"
)

var configCmd = &cobra.Command{
Use: "backup_yaml",
Short: "backup_yaml is a subcommand to check. It prints the current backup config file in yaml format to stdio.",

Run: func(cmd *cobra.Command, args []string) {
var params paramtable.BackupParams
params.GlobalInitWithYaml(config)
params.Init()

printParams(&params)
},
}

type YAMLConFig struct {
Log struct {
Level string `yaml:"level"`
Console bool `yaml:"console"`
File struct {
RootPath string `yaml:"rootPath"`
}
Http struct {
SimpleResponse bool `yaml:"simpleResponse"`
} `yaml:"http"`
} `yaml:"log"`
Milvus struct {
Address string `yaml:"address"`
Port int `yaml:"port"`
AuthorizationEnabled bool `yaml:"authorizationEnabled"`
TlsMode int `yaml:"tlsMode"`
User string `yaml:"user"`
Password string `yaml:"password"`
} `yaml:"milvus"`
Minio struct {
Address string `yaml:"address"`
Port int `yaml:"port"`
AccessKeyID string `yaml:"accessKeyID"`
secretAccessKey string `yaml:"secretAccessKey"`
UseSSL bool `yaml:"useSSL"`
UseIAM bool `yaml:"useIAM"`
CloudProvider string `yaml:"cloudProvider"`
IamEndpoint string `yaml:"iamEndpoint"`
BucketName string `yaml:"bucketName"`
RootPath string `yaml:"rootPath"`
BackupBucketName string `yaml:"backupBucketName"`
BackupRootPath string `yaml:"backupRootPath"`
} `yaml:"minio"`
Backup struct {
MaxSegmentGroupSize string `yaml:"maxSegmentGroupSize"`
} `yaml:"backup"`
}

func init() {
checkCmd.AddCommand(configCmd)
}

func printParams(base *paramtable.BackupParams) {

yml := YAMLConFig{}

yml.Log.Level = base.BaseTable.LoadWithDefault("log.level", "debug")
yml.Log.Console = base.ParseBool("log.console", false)
yml.Log.File.RootPath = base.LoadWithDefault("log.file.rootPath", "backup.log")

yml.Milvus.Address = base.LoadWithDefault("milvus.address", "localhost")
yml.Milvus.Port = base.ParseIntWithDefault("milvus.port", 19530)
yml.Milvus.AuthorizationEnabled = base.ParseBool("milvus.authorizationEnabled", false)
yml.Milvus.TlsMode = base.ParseIntWithDefault("milvus.tlsMode", 0)
yml.Milvus.User = base.BaseTable.LoadWithDefault("milvus.user", "")
yml.Milvus.Password = base.BaseTable.LoadWithDefault("milvus.password", "")

yml.Minio.Address = base.LoadWithDefault("minio.address", "localhost")
yml.Minio.Port = base.ParseIntWithDefault("minio.port", 9000)
yml.Minio.AccessKeyID = base.BaseTable.LoadWithDefault("minio.accessKeyID", "")
yml.Minio.secretAccessKey = base.BaseTable.LoadWithDefault("minio.secretAccessKey", "")
yml.Minio.UseSSL = base.ParseBool("minio.useSSL", false)
yml.Minio.UseIAM = base.ParseBool("minio.useIAM", false)
yml.Minio.CloudProvider = base.BaseTable.LoadWithDefault("minio.cloudProvider", "aws")
yml.Minio.IamEndpoint = base.BaseTable.LoadWithDefault("minio.iamEndpoint", "")
yml.Minio.BucketName = base.BaseTable.LoadWithDefault("minio.bucketName", "")
yml.Minio.RootPath = base.LoadWithDefault("minio.rootPath", "")
yml.Minio.BackupBucketName = base.LoadWithDefault("minio.backupBucketName", "")
yml.Minio.BackupRootPath = base.LoadWithDefault("minio.backupRootPath", "")

yml.Backup.MaxSegmentGroupSize = base.LoadWithDefault("backup.maxSegmentGroupSize", "5G")

bytes, err := yaml.Marshal(yml)
if err != nil {
panic(err)
}

fmt.Printf("%s\n%s", strings.Repeat("-", 80), string(bytes))
}
18 changes: 17 additions & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,15 @@ package cmd
import (
"errors"
"fmt"
"os"
"strings"

"github.com/spf13/cobra"
)

var (
config string
config string
yamlOverrides []string
)

var rootCmd = &cobra.Command{
Expand All @@ -18,14 +21,27 @@ var rootCmd = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) {
Error(cmd, args, errors.New("unrecognized command"))
},
PersistentPreRun: func(cmd *cobra.Command, args []string){
setEnvs(yamlOverrides)
},
}

func Execute() {
rootCmd.PersistentFlags().StringVarP(&config, "config", "", "backup.yaml", "config YAML file of milvus")
rootCmd.PersistentFlags().StringSliceVar(&yamlOverrides, "set", []string{}, "Override yaml values using a capitalized snake case format (--set MILVUS_USER=Marco)")
rootCmd.CompletionOptions.DisableDefaultCmd = true
rootCmd.Execute()
}

func SetVersionInfo(version, commit, date string) {
rootCmd.Version = fmt.Sprintf("%s (Built on %s from Git SHA %s)", version, date, commit)
}

// Set environment variables from yamlOverrides
func setEnvs(envs []string) {
for _, e := range envs {
env := strings.Split(e, "=")
os.Setenv(env[0], env[1])
}

}
2 changes: 1 addition & 1 deletion core/milvus_sdk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ func TestCreateIndex(t *testing.T) {
fmt.Println(schema)
client.CreateCollection(ctx, schema, 2)

idx, err := entity.NewIndexTRIE(entity.IP)
idx := entity.NewScalarIndex()
err = client.CreateIndex(ctx, _COLLECTION_NAME, _STR_FIELD_NAME, idx, false, gomilvus.WithIndexName("_default_idx_102"))
fmt.Println(err)

Expand Down
14 changes: 13 additions & 1 deletion core/paramtable/base_table_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package paramtable

import (
"os"
"testing"

"github.com/stretchr/testify/assert"
memkv "github.com/zilliztech/milvus-backup/internal/kv/mem"
"testing"
)

func TestParseDataSizeWithDefault(t *testing.T) {
Expand Down Expand Up @@ -37,3 +39,13 @@ func TestParseDataSizeWithDefault(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, size1024000, int64(1024000))
}

func TestTryLoadFromEnv(t *testing.T) {
base := &BaseTable{
params: memkv.NewMemoryKV(),
}
os.Setenv("MILVUS_USER", "Marco")
base.tryLoadFromEnv()

assert.Equal(t, "Marco", base.params.LoadWithDefault("milvus.user", ""))
}
6 changes: 4 additions & 2 deletions example/prepare_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,10 @@
hello_milvus.create_index("embeddings", index_params)


index_params2 = {"index_type": "Trie"}
hello_milvus2.create_index("var", index_params2)
hello_milvus2.create_index(field_name="var",index_name="scalar_index")

# index_params2 = {"index_type": "Trie"}
# hello_milvus2.create_index("var", index_params2)

print(f"Number of entities in hello_milvus2: {hello_milvus2.num_entities}") # check the num_entites

10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/google/uuid v1.1.2
github.com/json-iterator/go v1.1.12
github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
github.com/milvus-io/milvus-sdk-go/v2 v2.2.2
github.com/milvus-io/milvus-sdk-go/v2 v2.3.0
github.com/minio/minio-go/v7 v7.0.17
github.com/pkg/errors v0.9.1
github.com/sony/sonyflake v1.1.0
Expand All @@ -33,6 +33,7 @@ require (
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
google.golang.org/grpc v1.48.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v3 v3.0.1
)

require (
Expand Down Expand Up @@ -70,7 +71,7 @@ require (
github.com/magiconair/properties v1.8.5 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230522080721-2975bfe7a190 // indirect
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0 // indirect
github.com/minio/md5-simd v1.1.0 // indirect
github.com/minio/sha256-simd v0.1.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
Expand Down Expand Up @@ -101,10 +102,9 @@ require (
golang.org/x/tools v0.11.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 // indirect
google.golang.org/protobuf v1.28.1 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

replace github.com/milvus-io/milvus-sdk-go/v2 => github.com/wayblink/milvus-sdk-go/v2 v2.2.20
replace github.com/milvus-io/milvus-sdk-go/v2 => github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta5
12 changes: 6 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -383,8 +383,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230522080721-2975bfe7a190 h1:ZREJhOMgAvXs+K0ain51ibxAtCB8Lnn3EBZFHEykIlk=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230522080721-2975bfe7a190/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0 h1:t5CKm7+FXuD2rDLv/H8tpN9iY8F2dZvHF87xWBx8muU=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/minio-go/v7 v7.0.17 h1:5SiS3pqiQDbNhmXMxtqn2HzAInbN5cbHT7ip9F0F07E=
Expand Down Expand Up @@ -554,8 +554,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/wayblink/milvus-sdk-go/v2 v2.2.20 h1:kZCw4jL7AUpctlYfoXANtu+x4BFPHvBg6Xjps3PBUHM=
github.com/wayblink/milvus-sdk-go/v2 v2.2.20/go.mod h1:bFT+53/Uc+pWP85UJkxnWQs3H78q4JQd/39ed+BEGhU=
github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta5 h1:2hPosJlUEWFHVD85iNE0ywYhv/EIY4YK/0ZTsOzQE/E=
github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta5/go.mod h1:/o0HejRTQciRiHniFDMJiCHswVn1UbFChgtlNRRChd4=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
Expand Down Expand Up @@ -1002,8 +1002,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down

0 comments on commit f7fd49b

Please sign in to comment.