diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 256b2187..3660e61f 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -10,6 +10,9 @@ builds: goos: - linux - darwin + goarch: + - amd64 + - arm64 ldflags: - -w -s - -X main.version={{ .Version }} diff --git a/README.md b/README.md index 149409c3..11a21852 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -Romel # Milvus-backup Milvus-Backup is a tool that allows users to backup and restore Milvus data. This tool can be utilized either through the command line or an API server. diff --git a/cmd/backup_yaml.go b/cmd/backup_yaml.go new file mode 100644 index 00000000..20e44927 --- /dev/null +++ b/cmd/backup_yaml.go @@ -0,0 +1,102 @@ +package cmd + +import ( + "fmt" + "strings" + "github.com/spf13/cobra" + "github.com/zilliztech/milvus-backup/core/paramtable" + "gopkg.in/yaml.v3" +) + +var configCmd = &cobra.Command{ + Use: "backup_yaml", + Short: "backup_yaml is a subcommand to check. It prints the current backup config file in yaml format to stdio.", + + Run: func(cmd *cobra.Command, args []string) { + var params paramtable.BackupParams + params.GlobalInitWithYaml(config) + params.Init() + + printParams(¶ms) + }, +} + +type YAMLConFig struct { + Log struct { + Level string `yaml:"level"` + Console bool `yaml:"console"` + File struct { + RootPath string `yaml:"rootPath"` + } + Http struct { + SimpleResponse bool `yaml:"simpleResponse"` + } `yaml:"http"` + } `yaml:"log"` + Milvus struct { + Address string `yaml:"address"` + Port int `yaml:"port"` + AuthorizationEnabled bool `yaml:"authorizationEnabled"` + TlsMode int `yaml:"tlsMode"` + User string `yaml:"user"` + Password string `yaml:"password"` + } `yaml:"milvus"` + Minio struct { + Address string `yaml:"address"` + Port int `yaml:"port"` + AccessKeyID string `yaml:"accessKeyID"` + secretAccessKey string `yaml:"secretAccessKey"` + UseSSL bool `yaml:"useSSL"` + UseIAM bool `yaml:"useIAM"` + CloudProvider string `yaml:"cloudProvider"` + IamEndpoint string `yaml:"iamEndpoint"` + BucketName string `yaml:"bucketName"` + RootPath string `yaml:"rootPath"` + BackupBucketName string `yaml:"backupBucketName"` + BackupRootPath string `yaml:"backupRootPath"` + } `yaml:"minio"` + Backup struct { + MaxSegmentGroupSize string `yaml:"maxSegmentGroupSize"` + } `yaml:"backup"` +} + +func init() { + checkCmd.AddCommand(configCmd) +} + +func printParams(base *paramtable.BackupParams) { + + yml := YAMLConFig{} + + yml.Log.Level = base.BaseTable.LoadWithDefault("log.level", "debug") + yml.Log.Console = base.ParseBool("log.console", false) + yml.Log.File.RootPath = base.LoadWithDefault("log.file.rootPath", "backup.log") + + yml.Milvus.Address = base.LoadWithDefault("milvus.address", "localhost") + yml.Milvus.Port = base.ParseIntWithDefault("milvus.port", 19530) + yml.Milvus.AuthorizationEnabled = base.ParseBool("milvus.authorizationEnabled", false) + yml.Milvus.TlsMode = base.ParseIntWithDefault("milvus.tlsMode", 0) + yml.Milvus.User = base.BaseTable.LoadWithDefault("milvus.user", "") + yml.Milvus.Password = base.BaseTable.LoadWithDefault("milvus.password", "") + + yml.Minio.Address = base.LoadWithDefault("minio.address", "localhost") + yml.Minio.Port = base.ParseIntWithDefault("minio.port", 9000) + yml.Minio.AccessKeyID = base.BaseTable.LoadWithDefault("minio.accessKeyID", "") + yml.Minio.secretAccessKey = base.BaseTable.LoadWithDefault("minio.secretAccessKey", "") + yml.Minio.UseSSL = base.ParseBool("minio.useSSL", false) + yml.Minio.UseIAM = base.ParseBool("minio.useIAM", false) + yml.Minio.CloudProvider = base.BaseTable.LoadWithDefault("minio.cloudProvider", "aws") + yml.Minio.IamEndpoint = base.BaseTable.LoadWithDefault("minio.iamEndpoint", "") + yml.Minio.BucketName = base.BaseTable.LoadWithDefault("minio.bucketName", "") + yml.Minio.RootPath = base.LoadWithDefault("minio.rootPath", "") + yml.Minio.BackupBucketName = base.LoadWithDefault("minio.backupBucketName", "") + yml.Minio.BackupRootPath = base.LoadWithDefault("minio.backupRootPath", "") + + yml.Backup.MaxSegmentGroupSize = base.LoadWithDefault("backup.maxSegmentGroupSize", "5G") + + bytes, err := yaml.Marshal(yml) + if err != nil { + panic(err) + } + + fmt.Printf("%s\n%s", strings.Repeat("-", 80), string(bytes)) +} diff --git a/cmd/root.go b/cmd/root.go index 67d87952..3fc9939e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,12 +3,15 @@ package cmd import ( "errors" "fmt" + "os" + "strings" "github.com/spf13/cobra" ) var ( - config string + config string + yamlOverrides []string ) var rootCmd = &cobra.Command{ @@ -18,10 +21,14 @@ var rootCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { Error(cmd, args, errors.New("unrecognized command")) }, + PersistentPreRun: func(cmd *cobra.Command, args []string){ + setEnvs(yamlOverrides) + }, } func Execute() { rootCmd.PersistentFlags().StringVarP(&config, "config", "", "backup.yaml", "config YAML file of milvus") + rootCmd.PersistentFlags().StringSliceVar(&yamlOverrides, "set", []string{}, "Override yaml values using a capitalized snake case format (--set MILVUS_USER=Marco)") rootCmd.CompletionOptions.DisableDefaultCmd = true rootCmd.Execute() } @@ -29,3 +36,12 @@ func Execute() { func SetVersionInfo(version, commit, date string) { rootCmd.Version = fmt.Sprintf("%s (Built on %s from Git SHA %s)", version, date, commit) } + +// Set environment variables from yamlOverrides +func setEnvs(envs []string) { + for _, e := range envs { + env := strings.Split(e, "=") + os.Setenv(env[0], env[1]) + } + +} diff --git a/core/milvus_sdk_test.go b/core/milvus_sdk_test.go index 7f4eb675..ff3cf928 100644 --- a/core/milvus_sdk_test.go +++ b/core/milvus_sdk_test.go @@ -213,7 +213,7 @@ func TestCreateIndex(t *testing.T) { fmt.Println(schema) client.CreateCollection(ctx, schema, 2) - idx, err := entity.NewIndexTRIE(entity.IP) + idx := entity.NewScalarIndex() err = client.CreateIndex(ctx, _COLLECTION_NAME, _STR_FIELD_NAME, idx, false, gomilvus.WithIndexName("_default_idx_102")) fmt.Println(err) diff --git a/core/paramtable/base_table_test.go b/core/paramtable/base_table_test.go index 62042192..aac7cc44 100644 --- a/core/paramtable/base_table_test.go +++ b/core/paramtable/base_table_test.go @@ -1,9 +1,11 @@ package paramtable import ( + "os" + "testing" + "github.com/stretchr/testify/assert" memkv "github.com/zilliztech/milvus-backup/internal/kv/mem" - "testing" ) func TestParseDataSizeWithDefault(t *testing.T) { @@ -37,3 +39,13 @@ func TestParseDataSizeWithDefault(t *testing.T) { assert.NoError(t, err) assert.Equal(t, size1024000, int64(1024000)) } + +func TestTryLoadFromEnv(t *testing.T) { + base := &BaseTable{ + params: memkv.NewMemoryKV(), + } + os.Setenv("MILVUS_USER", "Marco") + base.tryLoadFromEnv() + + assert.Equal(t, "Marco", base.params.LoadWithDefault("milvus.user", "")) +} diff --git a/example/prepare_data.py b/example/prepare_data.py index 213a7c6d..ccd7eb1e 100644 --- a/example/prepare_data.py +++ b/example/prepare_data.py @@ -115,8 +115,10 @@ hello_milvus.create_index("embeddings", index_params) -index_params2 = {"index_type": "Trie"} -hello_milvus2.create_index("var", index_params2) +hello_milvus2.create_index(field_name="var",index_name="scalar_index") + +# index_params2 = {"index_type": "Trie"} +# hello_milvus2.create_index("var", index_params2) print(f"Number of entities in hello_milvus2: {hello_milvus2.num_entities}") # check the num_entites diff --git a/go.mod b/go.mod index f1c18c26..bda4cf4c 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/google/uuid v1.1.2 github.com/json-iterator/go v1.1.12 github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76 - github.com/milvus-io/milvus-sdk-go/v2 v2.2.2 + github.com/milvus-io/milvus-sdk-go/v2 v2.3.0 github.com/minio/minio-go/v7 v7.0.17 github.com/pkg/errors v0.9.1 github.com/sony/sonyflake v1.1.0 @@ -33,6 +33,7 @@ require ( golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 google.golang.org/grpc v1.48.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -70,7 +71,7 @@ require ( github.com/magiconair/properties v1.8.5 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/milvus-io/milvus-proto/go-api v0.0.0-20230522080721-2975bfe7a190 // indirect + github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0 // indirect github.com/minio/md5-simd v1.1.0 // indirect github.com/minio/sha256-simd v0.1.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -101,10 +102,9 @@ require ( golang.org/x/tools v0.11.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/milvus-io/milvus-sdk-go/v2 => github.com/wayblink/milvus-sdk-go/v2 v2.2.20 +replace github.com/milvus-io/milvus-sdk-go/v2 => github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta5 diff --git a/go.sum b/go.sum index daafe8f9..6ea338b8 100644 --- a/go.sum +++ b/go.sum @@ -383,8 +383,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/milvus-io/milvus-proto/go-api v0.0.0-20230522080721-2975bfe7a190 h1:ZREJhOMgAvXs+K0ain51ibxAtCB8Lnn3EBZFHEykIlk= -github.com/milvus-io/milvus-proto/go-api v0.0.0-20230522080721-2975bfe7a190/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk= +github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0 h1:t5CKm7+FXuD2rDLv/H8tpN9iY8F2dZvHF87xWBx8muU= +github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v7 v7.0.17 h1:5SiS3pqiQDbNhmXMxtqn2HzAInbN5cbHT7ip9F0F07E= @@ -554,8 +554,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/wayblink/milvus-sdk-go/v2 v2.2.20 h1:kZCw4jL7AUpctlYfoXANtu+x4BFPHvBg6Xjps3PBUHM= -github.com/wayblink/milvus-sdk-go/v2 v2.2.20/go.mod h1:bFT+53/Uc+pWP85UJkxnWQs3H78q4JQd/39ed+BEGhU= +github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta5 h1:2hPosJlUEWFHVD85iNE0ywYhv/EIY4YK/0ZTsOzQE/E= +github.com/wayblink/milvus-sdk-go/v2 v2.3.0-beta5/go.mod h1:/o0HejRTQciRiHniFDMJiCHswVn1UbFChgtlNRRChd4= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -1002,8 +1002,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=