From 9b3d816166eeb440d255d82fbe4ceb81c2a31551 Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 4 Jul 2024 15:01:44 +0900 Subject: [PATCH 001/118] multi target database --- core/writer/bigquery_handler.go | 182 +++++++++++++++++++++++ core/writer/channel_writer.go | 3 + core/writer/config_option.go | 68 +++++++++ core/writer/mysql_handler.go | 249 ++++++++++++++++++++++++++++++++ server/cdc_impl.go | 78 ++++++++++ server/model/common.go | 4 +- 6 files changed, 583 insertions(+), 1 deletion(-) create mode 100644 core/writer/bigquery_handler.go create mode 100644 core/writer/mysql_handler.go diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go new file mode 100644 index 00000000..a2b193cf --- /dev/null +++ b/core/writer/bigquery_handler.go @@ -0,0 +1,182 @@ +package writer + +import ( + "context" + "fmt" + "time" + + "cloud.google.com/go/bigquery" + "github.com/cenkalti/backoff/v4" + "github.com/cockroachdb/errors" + "go.uber.org/zap" + "google.golang.org/api/option" + + "github.com/zilliztech/milvus-cdc/core/api" + "github.com/zilliztech/milvus-cdc/core/config" + "github.com/zilliztech/milvus-cdc/core/log" +) + +type BigQueryDataHandler struct { + api.DataHandler + + projectID string + credentials string + connectTimeout int + client *bigquery.Client + retryOptions *backoff.ExponentialBackOff +} + +// NewBigQueryDataHandler options must include ProjectIDOption and CredentialsOption +func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*BigQueryDataHandler, error) { + handler := &BigQueryDataHandler{ + connectTimeout: 5, + retryOptions: backoff.NewExponentialBackOff(), + } + handler.retryOptions.MaxElapsedTime = 2 * time.Minute // 설정된 최대 재시도 시간 + + for _, option := range options { + option.Apply(handler) + } + if handler.projectID == "" { + return nil, errors.New("empty BigQuery project ID") + } + if handler.credentials == "" { + return nil, errors.New("empty BigQuery credentials") + } + + var err error + timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) + defer cancel() + + handler.client, err = handler.createBigQueryClient(timeoutContext) + if err != nil { + return nil, err + } + + return handler, nil +} + +func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { + client, err := bigquery.NewClient(ctx, m.projectID, option.WithCredentialsFile(m.credentials)) + if err != nil { + return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + } + return client, nil +} + +func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { + retryFunc := func() error { + q := m.client.Query(query) + q.Parameters = make([]bigquery.QueryParameter, 0, len(params)) + for k, v := range params { + q.Parameters = append(q.Parameters, bigquery.QueryParameter{Name: k, Value: v}) + } + job, err := q.Run(ctx) + if err != nil { + return err + } + status, err := job.Wait(ctx) + if err != nil { + return err + } + if err := status.Err(); err != nil { + return err + } + return nil + } + + err := backoff.Retry(retryFunc, backoff.WithContext(m.retryOptions, ctx)) + if err != nil { + log.Warn("retry operation failed", zap.Error(err)) + } + return err +} + +func (m *BigQueryDataHandler) CreateTable(ctx context.Context, param *api.CreateCollectionParam) error { + schema := bigquery.Schema{} + for _, field := range param.Schema.Fields { + schema = append(schema, &bigquery.FieldSchema{ + Name: field.Name, + Type: bigquery.FieldType(field.DataType), + }) + } + metaData := &bigquery.TableMetadata{ + Schema: schema, + } + tableRef := m.client.Dataset(param.Database).Table(param.Schema.CollectionName) + return tableRef.Create(ctx, metaData) +} + +func (m *BigQueryDataHandler) DropTable(ctx context.Context, param *api.DropCollectionParam) error { + tableRef := m.client.Dataset(param.Database).Table(param.CollectionName) + return tableRef.Delete(ctx) +} + +func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { + inserter := m.client.Dataset(param.Database).Table(param.CollectionName).Inserter() + var rows []*bigquery.ValuesSaver + var colName bigquery.Schema + var colValues []bigquery.Value + + for i, col := range param.Columns { + colName[i].Name = col.Name() + colValues[i] = col.FieldData() + } + + row := &bigquery.ValuesSaver{ + Schema: colName, + Row: colValues, + } + rows = append(rows, row) + + return inserter.Put(ctx, rows) +} + +func (m *BigQueryDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { + query := fmt.Sprintf("DELETE FROM `%s.%s` WHERE %s = @value", param.Database, param.CollectionName, param.Column.Name()) + params := map[string]interface{}{"value": param.Column.FieldData()} + return m.bigqueryOp(ctx, query, params) +} + +func (m *BigQueryDataHandler) CreateIndex(ctx context.Context, param *api.CreateIndexParam) error { + // BigQuery는 Vector Search Index만을 지원합니다. + query := fmt.Sprintf("CREATE OR REPLACE VECTOR INDEX `%s`.%s ON `%s`(embedding_v1) OPTIONS(distance_type='L2',index_type_type='IVF')", param.Database, param.IndexName, param.CollectionName) + return m.bigqueryOp(ctx, query, nil) +} + +func (m *BigQueryDataHandler) DropIndex(ctx context.Context, param *api.DropIndexParam) error { + // BigQuery는 인덱스를 지원하지 않습니다. + return nil +} + +func (m *BigQueryDataHandler) CreateDatabase(ctx context.Context, param *api.CreateDatabaseParam) error { + // BigQuery에서는 데이터셋을 생성하는 방식입니다. + dataset := m.client.Dataset(param.DbName) + return dataset.Create(ctx, &bigquery.DatasetMetadata{}) +} + +func (m *BigQueryDataHandler) DropDatabase(ctx context.Context, param *api.DropDatabaseParam) error { + // BigQuery에서는 데이터셋을 삭제하는 방식입니다. + dataset := m.client.Dataset(param.DbName) + return dataset.Delete(ctx) +} + +func (m *BigQueryDataHandler) DescribeTable(ctx context.Context, param *api.DescribeCollectionParam) error { + tableRef := m.client.Dataset(param.Database).Table(param.Name) + metaData, err := tableRef.Metadata(ctx) + if err != nil { + return err + } + fmt.Printf("Table: %s, Schema: %v\n", param.Name, metaData.Schema) + return nil +} + +func (m *BigQueryDataHandler) DescribeDatabase(ctx context.Context, param *api.DescribeDatabaseParam) error { + dataset := m.client.Dataset(param.Name) + metaData, err := dataset.Metadata(ctx) + if err != nil { + return err + } + fmt.Printf("Dataset: %s, Location: %s\n", param.Name, metaData.Location) + return nil +} diff --git a/core/writer/channel_writer.go b/core/writer/channel_writer.go index d5c465ef..59dbfff1 100644 --- a/core/writer/channel_writer.go +++ b/core/writer/channel_writer.go @@ -56,9 +56,11 @@ type ChannelWriter struct { partitionInfos util.Map[string, uint64] retryOptions []retry.Option + targetDBType string } func NewChannelWriter(dataHandler api.DataHandler, + targetDBType string, writerConfig config.WriterConfig, droppedObjs map[string]map[string]uint64, ) api.Writer { @@ -66,6 +68,7 @@ func NewChannelWriter(dataHandler api.DataHandler, dataHandler: dataHandler, messageManager: NewReplicateMessageManager(dataHandler, writerConfig.MessageBufferSize), retryOptions: util.GetRetryOptions(writerConfig.Retry), + targetDBType: targetDBType, } w.initAPIEventFuncs() w.initOPMessageFuncs() diff --git a/core/writer/config_option.go b/core/writer/config_option.go index d7bbcb74..8053edca 100644 --- a/core/writer/config_option.go +++ b/core/writer/config_option.go @@ -61,3 +61,71 @@ func DialConfigOption(dialConfig util.DialConfig) config.Option[*MilvusDataHandl object.dialConfig = dialConfig }) } + +func MilvusAddressOption(address string) config.Option[*MilvusDataHandler] { + return config.OptionFunc[*MilvusDataHandler](func(object *MilvusDataHandler) { + object.address = address + }) +} + +func MilvusUserOption(username string, password string) config.Option[*MilvusDataHandler] { + return config.OptionFunc[*MilvusDataHandler](func(object *MilvusDataHandler) { + object.username = username + object.password = password + }) +} + +func MilvusTLSOption(enable bool) config.Option[*MilvusDataHandler] { + return config.OptionFunc[*MilvusDataHandler](func(object *MilvusDataHandler) { + object.enableTLS = enable + }) +} + +func MilvusConnectTimeoutOption(timeout int) config.Option[*MilvusDataHandler] { + return config.OptionFunc[*MilvusDataHandler](func(object *MilvusDataHandler) { + if timeout > 0 { + object.connectTimeout = timeout + } + }) +} + +func MilvusIgnorePartitionOption(ignore bool) config.Option[*MilvusDataHandler] { + return config.OptionFunc[*MilvusDataHandler](func(object *MilvusDataHandler) { + object.ignorePartition = ignore + }) +} + +func MySQLAddressOption(address string) config.Option[*MySQLDataHandler] { + return config.OptionFunc[*MySQLDataHandler](func(object *MySQLDataHandler) { + object.address = address + }) +} + +func MySQLUserOption(username string, password string) config.Option[*MySQLDataHandler] { + return config.OptionFunc[*MySQLDataHandler](func(object *MySQLDataHandler) { + object.username = username + object.password = password + }) +} + +func MySQLConnectTimeoutOption(timeout int) config.Option[*MySQLDataHandler] { + return config.OptionFunc[*MySQLDataHandler](func(object *MySQLDataHandler) { + if timeout > 0 { + object.connectTimeout = timeout + } + }) +} + +func ProjectOption(projectID string) config.Option[*BigQueryDataHandler] { + return config.OptionFunc[*BigQueryDataHandler](func(object *BigQueryDataHandler) { + object.projectID = projectID + }) +} + +func BigQueryConnectTimeoutOption(timeout int) config.Option[*BigQueryDataHandler] { + return config.OptionFunc[*BigQueryDataHandler](func(object *BigQueryDataHandler) { + if timeout > 0 { + object.connectTimeout = timeout + } + }) +} diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go new file mode 100644 index 00000000..3a9d2868 --- /dev/null +++ b/core/writer/mysql_handler.go @@ -0,0 +1,249 @@ +package writer + +import ( + "context" + "database/sql" + "fmt" + "github.com/cenkalti/backoff/v4" + "github.com/go-sql-driver/mysql" + "github.com/milvus-io/milvus-sdk-go/v2/client" + "github.com/milvus-io/milvus-sdk-go/v2/entity" + "strings" + "time" + + "github.com/cockroachdb/errors" + "go.uber.org/zap" + + "github.com/zilliztech/milvus-cdc/core/api" + "github.com/zilliztech/milvus-cdc/core/config" + "github.com/zilliztech/milvus-cdc/core/log" +) + +type MySQLDataHandler struct { + api.DataHandler + + address string + username string + password string + connectTimeout int + db *sql.DB + retryOptions *backoff.ExponentialBackOff + mysqlCli *sql.DB +} + +func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDataHandler, error) { + handler := &MySQLDataHandler{ + connectTimeout: 5, + retryOptions: backoff.NewExponentialBackOff(), + } + handler.retryOptions.MaxElapsedTime = 2 * time.Minute // 설정된 최대 재시도 시간 + + for _, option := range options { + option.Apply(handler) + } + if handler.address == "" { + return nil, errors.New("empty MySQL address") + } + + var err error + timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) + defer cancel() + + handler.db, err = handler.createDBConnection(timeoutContext) + if err != nil { + return nil, err + } + + return handler, nil +} + +func (m *MySQLDataHandler) createDBConnection(ctx context.Context) (*sql.DB, error) { + connector, err := mysql.NewConnector(&mysql.Config{ + Addr: m.address, + User: m.username, + Passwd: m.password, + Net: "tcp", + }) + if err != nil { + return nil, fmt.Errorf("failed to create connector: %v", err) + } + + db := sql.OpenDB(connector) + + if err := db.PingContext(ctx); err != nil { + return nil, fmt.Errorf("failed to ping database: %v", err) + } + + return db, nil +} + +func (m *MySQLDataHandler) mysqlOp(ctx context.Context, query string, args ...interface{}) error { + retryFunc := func() error { + _, err := m.db.ExecContext(ctx, query, args...) + return err + } + + err := backoff.Retry(retryFunc, backoff.WithContext(m.retryOptions, ctx)) + if err != nil { + log.Warn("retry operation failed", zap.Error(err)) + } + return err +} + +func (m *MySQLDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { + query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", param.Schema.CollectionName, param.Schema.Fields) + return m.mysqlOp(ctx, query) +} + +func (m *MySQLDataHandler) DropCollection(ctx context.Context, param *api.DropCollectionParam) error { + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", param.CollectionName) + return m.mysqlOp(ctx, query) +} + +func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { + columns := []string{} + values := []string{} + for _, col := range param.Columns { + columns = append(columns, col.Name()) + values = append(values, "?") + } + query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", + param.CollectionName, join(columns, ","), join(values, ",")) + args := make([]interface{}, len(param.Columns)) + for i, col := range param.Columns { + args[i] = col.FieldData() + } + return m.mysqlOp(ctx, query, args...) +} + +func (m *MySQLDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { + query := fmt.Sprintf("DELETE FROM %s WHERE %s = ?", param.CollectionName, param.Column.Name) + return m.mysqlOp(ctx, query, param.Column.FieldData()) +} + +func (m *MySQLDataHandler) CreatePartition(ctx context.Context, param *api.CreatePartitionParam) error { + log.Warn("CreatePartition is not implemented in MySQL, please check it") + return nil +} + +func (m *MySQLDataHandler) DropPartition(ctx context.Context, param *api.DropPartitionParam) error { + log.Warn("DropPartition is not implemented in MySQL, please check it") + return nil +} + +func (m *MySQLDataHandler) CreateIndex(ctx context.Context, param *api.CreateIndexParam) error { + query := fmt.Sprintf("CREATE INDEX %s ON %s (%s)", + param.GetIndexName(), param.GetCollectionName(), param.GetFieldName()) + return m.mysqlOp(ctx, query) +} + +func (m *MySQLDataHandler) DropIndex(ctx context.Context, param *api.DropIndexParam) error { + query := fmt.Sprintf("DROP INDEX %s ON %s", param.IndexName, param.CollectionName) + return m.mysqlOp(ctx, query) +} + +func (m *MySQLDataHandler) LoadCollection(ctx context.Context, param *api.LoadCollectionParam) error { + log.Warn("LoadCollection is not implemented in MySQL, please check it") + return nil +} + +func (m *MySQLDataHandler) ReleaseCollection(ctx context.Context, param *api.ReleaseCollectionParam) error { + log.Warn("ReleaseCollection is not implemented in MySQL, please check it") + return nil +} + +func (m *MySQLDataHandler) LoadPartitions(ctx context.Context, param *api.LoadPartitionsParam) error { + log.Warn("LoadPartitions is not implemented in MySQL, please check it") + return nil +} + +func (m *MySQLDataHandler) ReleasePartitions(ctx context.Context, param *api.ReleasePartitionsParam) error { + log.Warn("ReleasePartitions is not implemented in MySQL, please check it") + return nil +} +func (m *MySQLDataHandler) Flush(ctx context.Context, param *api.FlushParam) error { + log.Warn("Flush is not implemented in MySQL, please check it") + return nil +} + +func (m *MySQLDataHandler) CreateDatabase(ctx context.Context, param *api.CreateDatabaseParam) error { + query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", param.DbName) + return m.mysqlOp(ctx, query) +} + +func (m *MySQLDataHandler) DropDatabase(ctx context.Context, param *api.DropDatabaseParam) error { + query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", param.DbName) + return m.mysqlOp(ctx, query) +} + +func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.ReplicateMessageParam) error { + var ( + resp *entity.MessageInfo + err error + opErr error + ) + opErr = m.mysqlOp(ctx, "", func(mysqlCli client.Client) error { + resp, err = mysqlCli.ReplicateMessage(ctx, param.ChannelName, + param.BeginTs, param.EndTs, + param.MsgsBytes, + param.StartPositions, param.EndPositions, + client.WithReplicateMessageMsgBase(param.Base)) + return err + }) + if err != nil { + return err + } + if opErr != nil { + return opErr + } + param.TargetMsgPosition = resp.Position + return nil +} + +func (m *MySQLDataHandler) DescribeCollection(ctx context.Context, param *api.DescribeCollectionParam) error { + query := fmt.Sprintf("DESCRIBE %s", param.Name) + return m.mysqlOp(ctx, query) +} + +func (m *MySQLDataHandler) DescribeDatabase(ctx context.Context, param *api.DescribeDatabaseParam) error { + query := "SHOW DATABASES" + rows, err := m.db.QueryContext(ctx, query) + if err != nil { + return err + } + defer rows.Close() + + var dbName string + for rows.Next() { + err := rows.Scan(&dbName) + if err != nil { + return err + } + if dbName == param.Name { + return nil + } + } + return errors.Newf("database [%s] not found", param.Name) +} + +func join(elems []string, sep string) string { + switch len(elems) { + case 0: + return "" + case 1: + return elems[0] + } + n := len(sep) * (len(elems) - 1) + for i := 0; i < len(elems); i++ { + n += len(elems[i]) + } + + var b strings.Builder + b.Grow(n) + b.WriteString(elems[0]) + for _, s := range elems[1:] { + b.WriteString(sep) + b.WriteString(s) + } + return b.String() +} diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 11171207..31a5fffa 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -328,6 +328,36 @@ func (e *MetaCDC) validCreateRequest(req *request.CreateRequest) error { return servererror.NewClientError("the rpc channel name is empty") } + if strings.ToLower(connectParam.TargetDBType) == "milvus" { + _, err := cdcwriter.NewMilvusDataHandler( + cdcwriter.MilvusAddressOption(fmt.Sprintf("%s:%d", connectParam.Host, connectParam.Port)), + cdcwriter.MilvusUserOption(connectParam.Username, connectParam.Password), + cdcwriter.MilvusTLSOption(connectParam.EnableTLS), + cdcwriter.MilvusIgnorePartitionOption(connectParam.IgnorePartition), + cdcwriter.MilvusConnectTimeoutOption(connectParam.ConnectTimeout)) + if err != nil { + log.Warn("fail to connect the milvus", zap.Any("connect_param", connectParam), zap.Error(err)) + return errors.WithMessage(err, "fail to connect the milvus") + } + } else if strings.ToLower(connectParam.TargetDBType) == "bigquery" { + _, err := cdcwriter.NewBigQueryDataHandler( + cdcwriter.ProjectOption(connectParam.ProjectId), + cdcwriter.BigQueryConnectTimeoutOption(connectParam.ConnectTimeout)) + if err != nil { + log.Warn("fail to connect the bigquery", zap.Any("connect_param", connectParam), zap.Error(err)) + return errors.WithMessage(err, "fail to connect the bigquery") + } + } else if strings.ToLower(connectParam.TargetDBType) == "mysql" { + _, err := cdcwriter.NewMySQLDataHandler( + cdcwriter.MySQLAddressOption(fmt.Sprintf("%s:%d", connectParam.Host, connectParam.Port)), + cdcwriter.MySQLUserOption(connectParam.Username, connectParam.Password), + cdcwriter.MySQLConnectTimeoutOption(connectParam.ConnectTimeout)) + if err != nil { + log.Warn("fail to connect the mysql", zap.Any("connect_param", connectParam), zap.Error(err)) + return errors.WithMessage(err, "fail to connect the mysql") + } + } + _, err := cdcwriter.NewMilvusDataHandler( cdcwriter.AddressOption(fmt.Sprintf("%s:%d", connectParam.Host, connectParam.Port)), cdcwriter.UserOption(connectParam.Username, connectParam.Password), @@ -336,6 +366,7 @@ func (e *MetaCDC) validCreateRequest(req *request.CreateRequest) error { cdcwriter.ConnectTimeoutOption(connectParam.ConnectTimeout), cdcwriter.DialConfigOption(connectParam.DialConfig), ) + if err != nil { log.Warn("fail to connect the milvus", zap.Any("connect_param", connectParam), zap.Error(err)) return errors.WithMessage(err, "fail to connect the milvus") @@ -565,6 +596,53 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err return nil, servererror.NewClientError("fail to create replicate channel manager") } targetConfig := milvusConnectParam + var writerObj api.Writer + + if strings.ToLower(targetConfig.TargetDBType) == "milvus" { + dataHandler, err := cdcwriter.NewMilvusDataHandler( + cdcwriter.MilvusAddressOption(fmt.Sprintf("%s:%d", targetConfig.Host, targetConfig.Port)), + cdcwriter.MilvusUserOption(targetConfig.Username, targetConfig.Password), + cdcwriter.MilvusTLSOption(targetConfig.EnableTLS), + cdcwriter.MilvusIgnorePartitionOption(targetConfig.IgnorePartition), + cdcwriter.MilvusConnectTimeoutOption(targetConfig.ConnectTimeout)) + + if err != nil { + taskLog.Warn("fail to new the data handler", zap.Error(err)) + return nil, servererror.NewClientError("fail to new the data handler, task_id: ") + } + writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ + MessageBufferSize: bufferSize, + Retry: e.config.Retry, + }, metaOp.GetAllDroppedObj()) + } else if strings.ToLower(targetConfig.TargetDBType) == "bigquery" { + dataHandler, err := cdcwriter.NewBigQueryDataHandler( + cdcwriter.ProjectOption(targetConfig.ProjectId), + cdcwriter.BigQueryConnectTimeoutOption(targetConfig.ConnectTimeout)) + + if err != nil { + taskLog.Warn("fail to new the data handler", zap.Error(err)) + return nil, servererror.NewClientError("fail to new the data handler, task_id: ") + } + writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ + MessageBufferSize: bufferSize, + Retry: e.config.Retry, + }, metaOp.GetAllDroppedObj()) + } else if strings.ToLower(targetConfig.TargetDBType) == "mysql" { + dataHandler, err := cdcwriter.NewMySQLDataHandler( + cdcwriter.MySQLAddressOption(fmt.Sprintf("%s:%d", targetConfig.Host, targetConfig.Port)), + cdcwriter.MySQLUserOption(targetConfig.Username, targetConfig.Password), + cdcwriter.MySQLConnectTimeoutOption(targetConfig.ConnectTimeout)) + + if err != nil { + taskLog.Warn("fail to new the data handler", zap.Error(err)) + return nil, servererror.NewClientError("fail to new the data handler, task_id: ") + } + writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ + MessageBufferSize: bufferSize, + Retry: e.config.Retry, + }, metaOp.GetAllDroppedObj()) + } + dataHandler, err := cdcwriter.NewMilvusDataHandler( cdcwriter.AddressOption(fmt.Sprintf("%s:%d", targetConfig.Host, targetConfig.Port)), cdcwriter.UserOption(targetConfig.Username, targetConfig.Password), diff --git a/server/model/common.go b/server/model/common.go index 5bab662a..6f6b8593 100644 --- a/server/model/common.go +++ b/server/model/common.go @@ -22,6 +22,7 @@ import "github.com/zilliztech/milvus-cdc/core/util" //go:generate easytags $GOFILE json,mapstructure type MilvusConnectParam struct { + TargetDBType string `json:"target_db_type" mapstructure:"target_db_type,omitempty"` Host string `json:"host" mapstructure:"host"` Port int `json:"port" mapstructure:"port"` Username string `json:"username,omitempty" mapstructure:"username,omitempty"` @@ -30,7 +31,8 @@ type MilvusConnectParam struct { DialConfig util.DialConfig `json:"dial_config" mapstructure:"dial_config"` IgnorePartition bool `json:"ignore_partition" mapstructure:"ignore_partition"` // ConnectTimeout unit: s - ConnectTimeout int `json:"connect_timeout" mapstructure:"connect_timeout"` + ConnectTimeout int `json:"connect_timeout" mapstructure:"connect_timeout"` + ProjectId string `json:"project_id" mapstructure:"project_id,omitempty"` } type CollectionInfo struct { From 90d0b514449a9d6c16a6feee9e61216f932b62cd Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 4 Jul 2024 17:02:52 +0900 Subject: [PATCH 002/118] multi target database --- server/cdc_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 31a5fffa..c5b356cf 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -655,7 +655,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err taskLog.Warn("fail to new the data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } - writerObj := cdcwriter.NewChannelWriter(dataHandler, config.WriterConfig{ + writerObj = cdcwriter.NewChannelWriter(dataHandler, targetConfig.TargetDBType, config.WriterConfig{ MessageBufferSize: bufferSize, Retry: e.config.Retry, }, metaOp.GetAllDroppedObj()) From 06e524e8ddfd5ca19a48af1162ccc409be719665 Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 4 Jul 2024 17:05:56 +0900 Subject: [PATCH 003/118] multi target database --- server/go.mod | 26 +++++++++++++++++++++++++- server/go.sum | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/server/go.mod b/server/go.mod index d01e8446..51347513 100644 --- a/server/go.mod +++ b/server/go.mod @@ -24,11 +24,19 @@ require ( ) require ( + cloud.google.com/go v0.110.0 // indirect + cloud.google.com/go/bigquery v1.50.0 // indirect + cloud.google.com/go/compute v1.19.1 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v0.13.0 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect github.com/AthenZ/athenz v1.10.39 // indirect github.com/DataDog/zstd v1.5.0 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect + github.com/apache/arrow/go/v11 v11.0.0 // indirect github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 // indirect + github.com/apache/thrift v0.16.0 // indirect github.com/ardielle/ardielle-go v1.5.2 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -57,8 +65,13 @@ require ( github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect + github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.7.1 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -68,13 +81,17 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/linkedin/goavro/v2 v2.11.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect + github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/minio/highwayhash v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -89,6 +106,7 @@ require ( github.com/pelletier/go-toml v1.9.3 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pierrec/lz4 v2.5.2+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -120,6 +138,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.etcd.io/etcd/api/v3 v3.5.5 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect @@ -127,6 +146,7 @@ require ( go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect go.etcd.io/etcd/raft/v3 v3.5.5 // indirect go.etcd.io/etcd/server/v3 v3.5.5 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0 // indirect go.opentelemetry.io/otel v1.13.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 // indirect @@ -141,13 +161,17 @@ require ( go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect + golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.12.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect diff --git a/server/go.sum b/server/go.sum index e8d5ff7f..ca164f4a 100644 --- a/server/go.sum +++ b/server/go.sum @@ -19,12 +19,15 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.50.0 h1:RscMV6LbnAmhAzD893Lv9nXXy2WCaJmbxYPWDLbGqNQ= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= @@ -32,6 +35,8 @@ cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2Aawl cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -71,7 +76,13 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/v11 v11.0.0 h1:hqauxvFQxww+0mEU/2XHG6LT7eZternCZq+A5Yly2uM= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= @@ -264,6 +275,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -302,6 +314,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -342,8 +356,12 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -437,12 +455,17 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -505,6 +528,10 @@ github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240604033341-0f2b1f29bf47 h1:4T github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240604033341-0f2b1f29bf47/go.mod h1:6ckCQ8h5iFncZcaIpGE8T8sqliwaw2Eu9UzHaCURMA4= github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A= github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -579,6 +606,8 @@ github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCr github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= @@ -757,6 +786,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= @@ -784,6 +815,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0 h1:g/BAN5o90Pr6D8xMRezjzGOHBpc15U+4oE53nZLiae4= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0/go.mod h1:+F41JBSkye7aYJELRvIMF0Z66reIwIOL0St75ZVwSJs= @@ -884,6 +917,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -964,6 +999,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1126,10 +1163,14 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1152,6 +1193,8 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= From ac522dbd0111da2f26af39ef8a6a96475df7a141 Mon Sep 17 00:00:00 2001 From: jisujisu1232 <48201571+jisujisu1232@users.noreply.github.com> Date: Thu, 4 Jul 2024 17:15:51 +0900 Subject: [PATCH 004/118] Delete .github directory --- .github/ISSUE_TEMPLATE/bug_report.yaml | 49 --- .../documentation_improvement.yaml | 34 -- .github/ISSUE_TEMPLATE/enhancement.yaml | 30 -- .github/ISSUE_TEMPLATE/feature-request.yaml | 36 -- .github/mergify.yml | 23 -- .github/workflows/ci.yaml | 316 ------------------ .github/workflows/golangci-lint.yml | 35 -- .github/workflows/image_publish.yml | 38 --- .github/workflows/release.yaml | 126 ------- .github/workflows/unit.yaml | 57 ---- 10 files changed, 744 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.yaml delete mode 100644 .github/ISSUE_TEMPLATE/documentation_improvement.yaml delete mode 100644 .github/ISSUE_TEMPLATE/enhancement.yaml delete mode 100644 .github/ISSUE_TEMPLATE/feature-request.yaml delete mode 100644 .github/mergify.yml delete mode 100644 .github/workflows/ci.yaml delete mode 100644 .github/workflows/golangci-lint.yml delete mode 100644 .github/workflows/image_publish.yml delete mode 100644 .github/workflows/release.yaml delete mode 100644 .github/workflows/unit.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml deleted file mode 100644 index b31d0e0a..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ /dev/null @@ -1,49 +0,0 @@ -name: 🐞 Bug report -description: Create a bug report to help us improve MilvusCDC -title: "[Bug]: " -body: -- type: markdown - attributes: - value: | - Thanks for taking the time to fill out this bug report! Please fill the form in English! -- type: textarea - attributes: - label: Current Behavior - description: A concise description of what you're experiencing. - placeholder: | - When I do , happens and I see the error message attached below: - ```...``` - validations: - required: true -- type: textarea - attributes: - label: Expected Behavior - description: A concise description of what you expected to happen. - placeholder: When I do , should happen instead. - validations: - required: false -- type: textarea - attributes: - label: Steps To Reproduce - description: Steps to reproduce the behavior. - placeholder: | - 1. In this environment... - 2. With this config... - 3. Run '...' - 4. See error... - validations: - required: false -- type: textarea - attributes: - label: Environment - description: | - Enter the Environment Details: - validations: - required: false -- type: textarea - attributes: - label: Anything else? - description: | - Links? References? Anything that will give us more context about the issue you are encountering! - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/documentation_improvement.yaml b/.github/ISSUE_TEMPLATE/documentation_improvement.yaml deleted file mode 100644 index b85ade86..00000000 --- a/.github/ISSUE_TEMPLATE/documentation_improvement.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: 📖 Documentation Improvement -description: Suggest improvements to our documentation -title: "[DOCS]: " -labels: [Documentation] -body: -- type: markdown - attributes: - value: | - Thanks for taking the time to fill out this documentation improvement request! -- type: textarea - attributes: - label: Documentation Link - description: Add a link to the page which needs improvement (if relevant) - validations: - required: false -- type: textarea - attributes: - label: Describe the problem - description: Is the documentation missing? Or is it confusing? Why is it confusing? - validations: - required: false -- type: textarea - attributes: - label: Describe the improvement - description: A clear and concise description of the improvement. - validations: - required: false -- type: textarea - attributes: - label: Anything else? - description: | - Links? References? Anything that will give us more context about the issue you are encountering! - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/enhancement.yaml b/.github/ISSUE_TEMPLATE/enhancement.yaml deleted file mode 100644 index 1f7bd6a6..00000000 --- a/.github/ISSUE_TEMPLATE/enhancement.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Enhancement request -description: As a developer, I want to make an enhancement for MilvusCDC -title: "[Enhancement]:" -labels: [kind/enhancement] -body: -- type: markdown - attributes: - value: | - Thanks for taking the time to request/suggest an enhancement for MilvusCDC! Please fill the form in English! -- type: textarea - attributes: - label: What would you like to be added? - description: A concise description of what you're expecting/suggesting. - placeholder: | - I would like to suggest/request a feature that's like... - validations: - required: false -- type: textarea - attributes: - label: Why is this needed? - description: A concise description of the reason/motivation - validations: - required: false -- type: textarea - attributes: - label: Anything else? - description: | - Links? References? Anything that will give us more context about this! - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/feature-request.yaml b/.github/ISSUE_TEMPLATE/feature-request.yaml deleted file mode 100644 index cc354428..00000000 --- a/.github/ISSUE_TEMPLATE/feature-request.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: 🚀 Feature request -description: As a user, I want to request a feature for MilvusCDC -title: "[Feature]:" -labels: [kind/feature] -body: -- type: markdown - attributes: - value: | - Thanks for taking the time to request a feature for MilvusCDC! Please fill the form in English! -- type: textarea - attributes: - label: Is your feature request related to a problem? Please describe. - description: A concise description of the problem you are facing or the motivation behind this feature request. - placeholder: | - I faced a problem due to which ... - validations: - required: false -- type: textarea - attributes: - label: Describe the solution you'd like. - description: A concise description of the solution for the issue. - validations: - required: false -- type: textarea - attributes: - label: Describe an alternate solution. - description: Is there any other approach to solve the problem? - validations: - required: false -- type: textarea - attributes: - label: Anything else? (Additional Context) - description: | - Links? References? Anything that will give us more context about this! - validations: - required: false diff --git a/.github/mergify.yml b/.github/mergify.yml deleted file mode 100644 index 88db7e64..00000000 --- a/.github/mergify.yml +++ /dev/null @@ -1,23 +0,0 @@ -pull_request_rules: - - name: Test passed for code changed - conditions: - - or: - - base=main - - "status-success=Unittest AMD64 Ubuntu 18.04" - - "status-success=lint" - actions: - label: - add: - - ci-passed - - - name: Remove ci-passed when some test failed - conditions: - - or: - - base=main - - or: - - "check-failure=Unittest AMD64 Ubuntu 18.04" - - "check-failure=lint" - actions: - label: - remove: - - ci-passed diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 545e53eb..00000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,316 +0,0 @@ -name: Test - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - milvus-cdc-system-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-go@v4 - with: - go-version: '1.21' - cache-dependency-path: server/go.sum - cache: true - - - name: set up python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - cache: 'pip' - - - name: Build CDC - timeout-minutes: 15 - shell: bash - run: | - docker build --build-arg GIT_COMMIT_ARG=$(git rev-parse --short HEA) -t milvus-cdc:latest . - docker tag milvus-cdc:latest milvusdb/milvus-cdc:latest - - - name: Creating kind cluster - uses: helm/kind-action@v1.2.0 - - - name: Print cluster information - run: | - kubectl config view - kubectl cluster-info - kubectl get nodes - kubectl get pods -o wide -n kube-system - helm version - kubectl version - - - name: Deploy Source Milvus - timeout-minutes: 15 - shell: bash - working-directory: tests/deployment/upstream/ - run: | - docker-compose up -d - bash ../../../scripts/check_healthy.sh - docker-compose ps -a - - - name: Deploy Downstream Milvus - timeout-minutes: 15 - shell: bash - working-directory: tests/deployment/downstream - run: | - helm repo add milvus https://zilliztech.github.io/milvus-helm - helm repo update - helm install --wait --timeout 720s cdc-downstream milvus/milvus -f standalone-values.yaml - kubectl get pods - kubectl port-forward service/cdc-downstream-milvus 19500:19530 >/dev/null 2>&1 & - sleep 20s - nc -vz 127.0.0.1 19500 - - - name: Deploy Milvus CDC - timeout-minutes: 15 - working-directory: deployment/docker - shell: bash - run: | - docker-compose up -d - sleep 20s - docker-compose ps - - - name: Create CDC task - timeout-minutes: 15 - run: | - curl --location '127.0.0.1:8444/cdc' \ - --header 'Content-Type: application/json' \ - --data '{ - "request_type": "create", - "request_data": { - "milvus_connect_param": { - "host": "127.0.0.1", - "port": 19500, - "username": "", - "password": "", - "enable_tls": false, - "connect_timeout": 120 - }, - "collection_infos": [ - { - "name": "*" - } - ], - "rpc_channel_info": { - "name": "by-dev-replicate-msg" - } - } - }' - - - name: Checkout Milvus Repo - uses: actions/checkout@v2 - with: - repository: zhuwenxing/milvus - ref: refine_checker - path: milvus - - - name: Run test - timeout-minutes: 15 - shell: bash - working-directory: milvus/tests/python_client - run: | - pip install -r requirements.txt --trusted-host https://test.pypi.org - cd chaos && pytest testcases/test_concurrent_operation.py --host 127.0.0.1 --log-cli-level=INFO --capture=no --request_duration 10m - - - name: List CDC task - timeout-minutes: 15 - working-directory: deployment/docker - shell: bash - run: | - docker-compose logs - curl --location '127.0.0.1:8444/cdc' \ - --header 'Content-Type: application/json' \ - --data '{ - "request_type": "list" - }' - - - name: Diff upstream and downstream - if: ${{ always() }} - timeout-minutes: 10 - shell: bash - working-directory: milvus/tests/python_client/chaos/scripts - run: | - python diff_upstream_downstream.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500 - - - - name: Export milvus cdc logs - if: ${{ always() }} - timeout-minutes: 5 - working-directory: deployment/docker - run: | - docker-compose ps -a - docker stats --no-stream - docker-compose logs > server.log - - - name: Export upstream milvus logs - if: ${{ always() }} - timeout-minutes: 5 - working-directory: tests/deployment/upstream - run: | - docker-compose ps -a - docker stats --no-stream - bash ../../../scripts/export_log_docker.sh - - name: Export downstream milvus logs - if: ${{ always() }} - timeout-minutes: 5 - working-directory: scripts - run: | - kubectl get pods || true - bash export_log_k8s.sh default cdc-downstream k8s_logs - - - name: Upload logs - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: system-test-logs - path: | - scripts/k8s_logs - tests/deployment/upstream/logs - deployment/docker/server.log - - milvus-cdc-function-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-go@v4 - with: - go-version: '1.21' - cache-dependency-path: server/go.sum - cache: true - - - name: set up python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - cache: 'pip' - - - name: Build CDC - timeout-minutes: 15 - working-directory: server - shell: bash - run: | - go build -o milvus-cdc main/main.go - ls -l - - - name: Creating kind cluster - uses: helm/kind-action@v1.2.0 - - - name: Print cluster information - run: | - kubectl config view - kubectl cluster-info - kubectl get nodes - kubectl get pods -o wide -n kube-system - helm version - kubectl version - - - name: Deploy Source Milvus - timeout-minutes: 15 - shell: bash - working-directory: tests/deployment/upstream/ - run: | - docker-compose up -d - bash ../../../scripts/check_healthy.sh - docker-compose ps -a - - - name: Deploy Downstream Milvus - timeout-minutes: 15 - shell: bash - working-directory: tests/deployment/downstream - run: | - helm repo add milvus https://zilliztech.github.io/milvus-helm - helm repo update - helm install --wait --timeout 720s cdc-downstream milvus/milvus -f standalone-values.yaml - kubectl get pods - kubectl port-forward service/cdc-downstream-milvus 19500:19530 >/dev/null 2>&1 & - sleep 20s - nc -vz 127.0.0.1 19500 - - - name: Deploy Milvus CDC - timeout-minutes: 15 - working-directory: server - shell: bash - run: | - ./milvus-cdc > server.log 2>&1 & - sleep 20s - - - name: Create CDC task - timeout-minutes: 15 - run: | - curl --location '127.0.0.1:8444/cdc' \ - --header 'Content-Type: application/json' \ - --data '{ - "request_type": "create", - "request_data": { - "milvus_connect_param": { - "host": "127.0.0.1", - "port": 19500, - "username": "", - "password": "", - "enable_tls": false, - "connect_timeout": 120 - }, - "collection_infos": [ - { - "name": "*" - } - ], - "rpc_channel_info": { - "name": "by-dev-replicate-msg" - } - } - }' - - - name: Run test - timeout-minutes: 15 - shell: bash - working-directory: tests - run: | - pip install -r requirements.txt --trusted-host https://test.pypi.org - pytest testcases/test_cdc_sync_requests.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500 - - - name: List CDC task - if: ${{ always() }} - timeout-minutes: 15 - working-directory: server - shell: bash - run: | - cat server.log | tail -n 100 - curl --location '127.0.0.1:8444/cdc' \ - --header 'Content-Type: application/json' \ - --data '{ - "request_type": "list" - }' - - - name: Export upstream milvus logs - if: ${{ always() }} - timeout-minutes: 5 - working-directory: tests/deployment/upstream - run: | - docker-compose ps -a - docker stats --no-stream - bash ../../../scripts/export_log_docker.sh - - name: Export downstream milvus logs - if: ${{ always() }} - timeout-minutes: 5 - working-directory: scripts - run: | - kubectl get pods || true - bash export_log_k8s.sh default cdc-downstream k8s_logs - - - name: Upload logs - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: func-test-logs - path: | - scripts/k8s_logs - tests/deployment/upstream/logs - server/server.log diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml deleted file mode 100644 index fa800b1d..00000000 --- a/.github/workflows/golangci-lint.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: golangci-lint -on: - push: - tags: - - v* - branches: - - main - pull_request: -permissions: - contents: read - # Optional: allow read access to pull request. Use with `only-new-issues` option. - # pull-requests: read -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: "1.21" - - name: golangci-lint core - uses: golangci/golangci-lint-action@v6 - with: - version: latest - working-directory: ./core - args: --timeout=5m --out-format=colored-line-number ./... - skip-cache: true - - name: golangci-lint server - uses: golangci/golangci-lint-action@v6 - with: - version: latest - working-directory: ./server - args: --timeout=5m --out-format=colored-line-number ./... - skip-cache: true \ No newline at end of file diff --git a/.github/workflows/image_publish.yml b/.github/workflows/image_publish.yml deleted file mode 100644 index 7ba281cd..00000000 --- a/.github/workflows/image_publish.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Build and Push Docker image - -on: - push: - branches: - - main - pull_request: - branches: - - main - types: [closed] - -jobs: - build: - runs-on: ubuntu-latest - if: github.event.pull_request.merged == true || github.event_name == 'push' - steps: - - uses: actions/checkout@v3 - - run: git fetch --force --tags - - uses: actions/setup-go@v4 - with: - go-version: '1.21' - - name: Docker Login - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Build and push docker image - run: | - bash build_image.sh - BRANCH=$(git symbolic-ref --short HEAD) - COMMIT_ID=$(git rev-parse --short HEAD) - docker tag milvus-cdc:${BRANCH}-${COMMIT_ID} ${{ secrets.DOCKER_USER }}/milvus-cdc:${BRANCH}-${COMMIT_ID} - docker push ${{ secrets.DOCKER_USER }}/milvus-cdc:${BRANCH}-${COMMIT_ID} - docker tag milvus-cdc:${BRANCH}-${COMMIT_ID} ${{ secrets.DOCKER_USER }}/milvus-cdc:${BRANCH}-latest - docker push ${{ secrets.DOCKER_USER }}/milvus-cdc:${BRANCH}-latest - - - diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 465713ac..00000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,126 +0,0 @@ -name: release - -on: - push: - tags: - - 'v*' - workflow_dispatch: - -permissions: - contents: write - -jobs: - build-linux-binary: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - run: git fetch --force --tags - - uses: actions/setup-go@v4 - with: - go-version: '1.21' - cache-dependency-path: server/go.sum - cache: true - - - uses: goreleaser/goreleaser-action@v4 - with: - # either 'goreleaser' (default) or 'goreleaser-pro': - distribution: goreleaser - workdir: ./server - version: latest - args: release --skip-publish --config .goreleaser-linux.yaml - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} - - name: Upload - uses: actions/upload-artifact@v3 - with: - name: milvus-cdc-linux - path: | - server/dist/milvus-cdc* - server/dist/checksums.txt - - build-darwin-binary: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-go@v4 - with: - go-version: '1.21' - cache-dependency-path: server/go.sum - cache: true - - - uses: goreleaser/goreleaser-action@v4 - with: - # either 'goreleaser' (default) or 'goreleaser-pro': - distribution: goreleaser - workdir: ./server - version: latest - args: release --skip-publish --config .goreleaser-darwin.yaml - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} - - name: Upload - uses: actions/upload-artifact@v3 - with: - name: milvus-cdc-darwin - path: | - server/dist/milvus-cdc* - server/dist/checksums.txt - merge-and-release: - needs: [build-linux-binary, build-darwin-binary] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - run: git fetch --force --tags - - uses: actions/setup-go@v4 - with: - go-version: '1.21' - cache-dependency-path: server/go.sum - cache: true - - name: Make directories - run: | - mkdir -p ./milvus-cdc-build/linux - mkdir -p ./milvus-cdc-build/darwin - - name: Download linux binaries - uses: actions/download-artifact@v3 - with: - name: milvus-cdc-linux - path: ./milvus-cdc-build/linux - - name: Download darwin binaries - uses: actions/download-artifact@v3 - with: - name: milvus-cdc-darwin - path: ./milvus-cdc-build/darwin - - name: Merge checksum file - run: | - cd ./milvus-cdc-build - cat ./darwin/checksums.txt >> checksums.txt - cat ./linux/checksums.txt >> checksums.txt - rm ./darwin/checksums.txt - rm ./linux/checksums.txt - - name: Check git status - run: | - tree - git status - - - name: Release - uses: goreleaser/goreleaser-action@v4 - with: - args: release --clean --config .goreleaser.yaml - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} - - name: Docker Login - uses: docker/login-action@v2 - with: - username: milvusdb - password: ${{ secrets.DOCKER_TOKEN }} - - name: Build and push docker image - run: | - tag=$(git tag --sort=-version:refname | head -1) - docker build -t milvusdb/milvus-cdc:$tag -t milvusdb/milvus-cdc:latest . - docker push milvusdb/milvus-cdc:$tag - docker push milvusdb/milvus-cdc:latest \ No newline at end of file diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml deleted file mode 100644 index 03c84a45..00000000 --- a/.github/workflows/unit.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: Unittest - -on: - push: - paths: - - 'core/**' - - 'server/**' - - 'tests/**' - # Triggers the workflow on push or pull request events but only for the master branch - pull_request: - paths: - - 'core/**' - - 'server/**' - - 'tests/**' - - -jobs: - # This workflow contains a single job called "build" - build: - name: Unittest AMD64 Ubuntu ${{ matrix.ubuntu }} - # The type of runner that the job will run on - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - ubuntu: [18.04] - env: - UBUNTU: ${{ matrix.ubuntu }} - services: - mysql: - image: mysql:5.7.42 - env: - MYSQL_ROOT_PASSWORD: 123456 - MYSQL_DATABASE: milvuscdc - ports: - - 3306:3306 - etcd: - image: quay.io/coreos/etcd:v3.5.5 - env: - ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379 - ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 - ports: - - 2379:2379 - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 - # Runs a single command using the runners shell - - name: Run Unittest - run: make test-go - - name: Upload coverage to Codecov - if: github.repository == 'zilliztech/milvus-cdc' - uses: codecov/codecov-action@v3.1.1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.project.out - name: ubuntu-${{ matrix.ubuntu }}-unittests \ No newline at end of file From 3e48afe7f9bde20370cad597dc93932aaf285327 Mon Sep 17 00:00:00 2001 From: ian-lss <38341885+ian-lss@users.noreply.github.com> Date: Fri, 5 Jul 2024 15:13:38 +0900 Subject: [PATCH 005/118] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 34e987fd..3316a196 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ CDC is "Change Data Capture", and Milvus-CDC is a change data capture tool for M You can download the executable files in the [release page](https://github.com/zilliztech/milvus-cdc/releases) or compile the source code to get the cdc tool. -- how to compile source code? +- how to compile source code? ```bash git clone https://github.com/zilliztech/milvus-cdc.git From 6a6c3dda2ec9081fff628ce5beffde6648961f7f Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 5 Jul 2024 15:41:23 +0900 Subject: [PATCH 006/118] multi target database --- server/cdc_impl.go | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index c5b356cf..4fd9f348 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -328,8 +328,9 @@ func (e *MetaCDC) validCreateRequest(req *request.CreateRequest) error { return servererror.NewClientError("the rpc channel name is empty") } + var err error if strings.ToLower(connectParam.TargetDBType) == "milvus" { - _, err := cdcwriter.NewMilvusDataHandler( + _, err = cdcwriter.NewMilvusDataHandler( cdcwriter.MilvusAddressOption(fmt.Sprintf("%s:%d", connectParam.Host, connectParam.Port)), cdcwriter.MilvusUserOption(connectParam.Username, connectParam.Password), cdcwriter.MilvusTLSOption(connectParam.EnableTLS), @@ -340,7 +341,7 @@ func (e *MetaCDC) validCreateRequest(req *request.CreateRequest) error { return errors.WithMessage(err, "fail to connect the milvus") } } else if strings.ToLower(connectParam.TargetDBType) == "bigquery" { - _, err := cdcwriter.NewBigQueryDataHandler( + _, err = cdcwriter.NewBigQueryDataHandler( cdcwriter.ProjectOption(connectParam.ProjectId), cdcwriter.BigQueryConnectTimeoutOption(connectParam.ConnectTimeout)) if err != nil { @@ -348,7 +349,7 @@ func (e *MetaCDC) validCreateRequest(req *request.CreateRequest) error { return errors.WithMessage(err, "fail to connect the bigquery") } } else if strings.ToLower(connectParam.TargetDBType) == "mysql" { - _, err := cdcwriter.NewMySQLDataHandler( + _, err = cdcwriter.NewMySQLDataHandler( cdcwriter.MySQLAddressOption(fmt.Sprintf("%s:%d", connectParam.Host, connectParam.Port)), cdcwriter.MySQLUserOption(connectParam.Username, connectParam.Password), cdcwriter.MySQLConnectTimeoutOption(connectParam.ConnectTimeout)) @@ -358,19 +359,11 @@ func (e *MetaCDC) validCreateRequest(req *request.CreateRequest) error { } } - _, err := cdcwriter.NewMilvusDataHandler( - cdcwriter.AddressOption(fmt.Sprintf("%s:%d", connectParam.Host, connectParam.Port)), - cdcwriter.UserOption(connectParam.Username, connectParam.Password), - cdcwriter.TLSOption(connectParam.EnableTLS), - cdcwriter.IgnorePartitionOption(connectParam.IgnorePartition), - cdcwriter.ConnectTimeoutOption(connectParam.ConnectTimeout), - cdcwriter.DialConfigOption(connectParam.DialConfig), - ) - if err != nil { - log.Warn("fail to connect the milvus", zap.Any("connect_param", connectParam), zap.Error(err)) - return errors.WithMessage(err, "fail to connect the milvus") + log.Warn("fail to connect the ", zap.String("targetDBType", connectParam.TargetDBType), zap.Any("connect_param", connectParam), zap.Error(err)) + return errors.WithMessage(err, "fail to connect the "+connectParam.TargetDBType) } + return nil } @@ -643,22 +636,10 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err }, metaOp.GetAllDroppedObj()) } - dataHandler, err := cdcwriter.NewMilvusDataHandler( - cdcwriter.AddressOption(fmt.Sprintf("%s:%d", targetConfig.Host, targetConfig.Port)), - cdcwriter.UserOption(targetConfig.Username, targetConfig.Password), - cdcwriter.TLSOption(targetConfig.EnableTLS), - cdcwriter.IgnorePartitionOption(targetConfig.IgnorePartition), - cdcwriter.ConnectTimeoutOption(targetConfig.ConnectTimeout), - cdcwriter.DialConfigOption(targetConfig.DialConfig), - ) if err != nil { taskLog.Warn("fail to new the data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } - writerObj = cdcwriter.NewChannelWriter(dataHandler, targetConfig.TargetDBType, config.WriterConfig{ - MessageBufferSize: bufferSize, - Retry: e.config.Retry, - }, metaOp.GetAllDroppedObj()) e.replicateEntityMap.Lock() defer e.replicateEntityMap.Unlock() From af9254ca8c524232356502ebd68d7390b0f18ac1 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 5 Jul 2024 17:46:20 +0900 Subject: [PATCH 007/118] multi target database --- core/writer/mysql_handler.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 3a9d2868..1057f72f 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -59,10 +59,11 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat func (m *MySQLDataHandler) createDBConnection(ctx context.Context) (*sql.DB, error) { connector, err := mysql.NewConnector(&mysql.Config{ - Addr: m.address, - User: m.username, - Passwd: m.password, - Net: "tcp", + Addr: m.address, + User: m.username, + Passwd: m.password, + Net: "tcp", + AllowNativePasswords: true, }) if err != nil { return nil, fmt.Errorf("failed to create connector: %v", err) From e3640917fc1da882febcfee769f8e31135a0b952 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 5 Jul 2024 18:22:38 +0900 Subject: [PATCH 008/118] multi target database --- core/writer/mysql_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 1057f72f..1218e872 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -64,6 +64,7 @@ func (m *MySQLDataHandler) createDBConnection(ctx context.Context) (*sql.DB, err Passwd: m.password, Net: "tcp", AllowNativePasswords: true, + DBName: "information_schema", }) if err != nil { return nil, fmt.Errorf("failed to create connector: %v", err) From c1aebbd31f0d92997c813b80245968f067415bf4 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 5 Jul 2024 19:29:29 +0900 Subject: [PATCH 009/118] multi target database --- server/cdc_impl.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 4fd9f348..90da3700 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -359,11 +359,6 @@ func (e *MetaCDC) validCreateRequest(req *request.CreateRequest) error { } } - if err != nil { - log.Warn("fail to connect the ", zap.String("targetDBType", connectParam.TargetDBType), zap.Any("connect_param", connectParam), zap.Error(err)) - return errors.WithMessage(err, "fail to connect the "+connectParam.TargetDBType) - } - return nil } @@ -600,7 +595,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err cdcwriter.MilvusConnectTimeoutOption(targetConfig.ConnectTimeout)) if err != nil { - taskLog.Warn("fail to new the data handler", zap.Error(err)) + taskLog.Warn("fail to new the milvus data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ @@ -613,7 +608,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err cdcwriter.BigQueryConnectTimeoutOption(targetConfig.ConnectTimeout)) if err != nil { - taskLog.Warn("fail to new the data handler", zap.Error(err)) + taskLog.Warn("fail to new the bigquery data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ @@ -627,7 +622,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err cdcwriter.MySQLConnectTimeoutOption(targetConfig.ConnectTimeout)) if err != nil { - taskLog.Warn("fail to new the data handler", zap.Error(err)) + taskLog.Warn("fail to new the mysql data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ @@ -636,11 +631,6 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err }, metaOp.GetAllDroppedObj()) } - if err != nil { - taskLog.Warn("fail to new the data handler", zap.Error(err)) - return nil, servererror.NewClientError("fail to new the data handler, task_id: ") - } - e.replicateEntityMap.Lock() defer e.replicateEntityMap.Unlock() entity, ok := e.replicateEntityMap.data[milvusAddress] From 32ed9186bd57db4c88ddf8b193e38ea6bbb86aa0 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 5 Jul 2024 19:38:08 +0900 Subject: [PATCH 010/118] multi target database --- core/writer/mysql_handler.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 1218e872..60fa0565 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -46,10 +46,8 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat } var err error - timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) - defer cancel() - handler.db, err = handler.createDBConnection(timeoutContext) + handler.db, err = handler.createDBConnection(handler.connectTimeout) if err != nil { return nil, err } @@ -57,7 +55,7 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat return handler, nil } -func (m *MySQLDataHandler) createDBConnection(ctx context.Context) (*sql.DB, error) { +func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, error) { connector, err := mysql.NewConnector(&mysql.Config{ Addr: m.address, User: m.username, @@ -65,6 +63,7 @@ func (m *MySQLDataHandler) createDBConnection(ctx context.Context) (*sql.DB, err Net: "tcp", AllowNativePasswords: true, DBName: "information_schema", + Timeout: time.Duration(connectionTimeout) * time.Second, }) if err != nil { return nil, fmt.Errorf("failed to create connector: %v", err) @@ -72,8 +71,8 @@ func (m *MySQLDataHandler) createDBConnection(ctx context.Context) (*sql.DB, err db := sql.OpenDB(connector) - if err := db.PingContext(ctx); err != nil { - return nil, fmt.Errorf("failed to ping database: %v", err) + if err := db.Ping(); err != nil { + return nil, fmt.Errorf("failed to ping database with dbconn: %v", err) } return db, nil From 5962ccb778a7a71a0e6954aff2495a102dd79928 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 5 Jul 2024 19:57:25 +0900 Subject: [PATCH 011/118] multi target database --- core/writer/mysql_handler.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 60fa0565..190e203f 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -72,7 +72,7 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e db := sql.OpenDB(connector) if err := db.Ping(); err != nil { - return nil, fmt.Errorf("failed to ping database with dbconn: %v", err) + return nil, fmt.Errorf("failed to ping mysql database : %v", err) } return db, nil @@ -119,6 +119,7 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e func (m *MySQLDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { query := fmt.Sprintf("DELETE FROM %s WHERE %s = ?", param.CollectionName, param.Column.Name) + log.Info(query) return m.mysqlOp(ctx, query, param.Column.FieldData()) } @@ -135,11 +136,13 @@ func (m *MySQLDataHandler) DropPartition(ctx context.Context, param *api.DropPar func (m *MySQLDataHandler) CreateIndex(ctx context.Context, param *api.CreateIndexParam) error { query := fmt.Sprintf("CREATE INDEX %s ON %s (%s)", param.GetIndexName(), param.GetCollectionName(), param.GetFieldName()) + log.Info(query) return m.mysqlOp(ctx, query) } func (m *MySQLDataHandler) DropIndex(ctx context.Context, param *api.DropIndexParam) error { query := fmt.Sprintf("DROP INDEX %s ON %s", param.IndexName, param.CollectionName) + log.Info(query) return m.mysqlOp(ctx, query) } @@ -169,11 +172,13 @@ func (m *MySQLDataHandler) Flush(ctx context.Context, param *api.FlushParam) err func (m *MySQLDataHandler) CreateDatabase(ctx context.Context, param *api.CreateDatabaseParam) error { query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", param.DbName) + log.Info(query) return m.mysqlOp(ctx, query) } func (m *MySQLDataHandler) DropDatabase(ctx context.Context, param *api.DropDatabaseParam) error { query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", param.DbName) + log.Info(query) return m.mysqlOp(ctx, query) } @@ -183,6 +188,8 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl err error opErr error ) + log.Info("ReplicateMessage", zap.Any("param", param)) + opErr = m.mysqlOp(ctx, "", func(mysqlCli client.Client) error { resp, err = mysqlCli.ReplicateMessage(ctx, param.ChannelName, param.BeginTs, param.EndTs, @@ -203,11 +210,13 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl func (m *MySQLDataHandler) DescribeCollection(ctx context.Context, param *api.DescribeCollectionParam) error { query := fmt.Sprintf("DESCRIBE %s", param.Name) + log.Info(query) return m.mysqlOp(ctx, query) } func (m *MySQLDataHandler) DescribeDatabase(ctx context.Context, param *api.DescribeDatabaseParam) error { query := "SHOW DATABASES" + log.Info(query) rows, err := m.db.QueryContext(ctx, query) if err != nil { return err From 4ed5ca36ecc088256a1fd7143d0868ac5d5ce9cf Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 8 Jul 2024 17:26:49 +0900 Subject: [PATCH 012/118] multi target database --- core/writer/mysql_handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 190e203f..85ca194c 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -43,6 +43,8 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat } if handler.address == "" { return nil, errors.New("empty MySQL address") + } else { + log.Info("MySQL address", zap.String("address", handler.address)) } var err error From 4efa7bc6157c157426c965fbc26410f2d8006d22 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 8 Jul 2024 17:38:58 +0900 Subject: [PATCH 013/118] multi target database --- core/writer/mysql_handler.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 85ca194c..7e1d9454 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -72,6 +72,10 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e } db := sql.OpenDB(connector) + if err := db.QueryRow("use information_schema"); err != nil { + return nil, fmt.Errorf("failed to use mysql selected database : %v", err) + + } if err := db.Ping(); err != nil { return nil, fmt.Errorf("failed to ping mysql database : %v", err) From f216b62be008a0c0d5ea13b5c13f4043ecc40102 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 8 Jul 2024 17:46:20 +0900 Subject: [PATCH 014/118] multi target database --- core/writer/mysql_handler.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 7e1d9454..0560e4ae 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -43,8 +43,6 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat } if handler.address == "" { return nil, errors.New("empty MySQL address") - } else { - log.Info("MySQL address", zap.String("address", handler.address)) } var err error @@ -72,14 +70,6 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e } db := sql.OpenDB(connector) - if err := db.QueryRow("use information_schema"); err != nil { - return nil, fmt.Errorf("failed to use mysql selected database : %v", err) - - } - - if err := db.Ping(); err != nil { - return nil, fmt.Errorf("failed to ping mysql database : %v", err) - } return db, nil } From fd04b586c9d7da97a4c4070eb5328db6651f38a3 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 9 Jul 2024 11:15:38 +0900 Subject: [PATCH 015/118] multi target database --- core/writer/mysql_handler.go | 6 +++++- server/cdc_impl.go | 29 ++++++++++++++++++----------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 0560e4ae..5421c050 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -70,7 +70,11 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e } db := sql.OpenDB(connector) - + /* + if err := db.Ping(); err != nil { + return nil, fmt.Errorf("failed to ping mysql database : %v", err) + } + */ return db, nil } diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 90da3700..123fb454 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -529,20 +529,27 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err milvusConnectParam := info.MilvusConnectParam milvusAddress := fmt.Sprintf("%s:%d", milvusConnectParam.Host, milvusConnectParam.Port) + var milvusClient api.TargetAPI + var err error + ctx := context.TODO() timeoutCtx, cancelFunc := context.WithTimeout(ctx, time.Duration(milvusConnectParam.ConnectTimeout)*time.Second) - milvusClient, err := cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ - Address: milvusAddress, - Username: milvusConnectParam.Username, - Password: milvusConnectParam.Password, - EnableTLS: milvusConnectParam.EnableTLS, - DialConfig: milvusConnectParam.DialConfig, - }) - cancelFunc() - if err != nil { - taskLog.Warn("fail to new target", zap.String("address", milvusAddress), zap.Error(err)) - return nil, servererror.NewClientError("fail to connect target milvus server") + if strings.ToLower(milvusConnectParam.TargetDBType) == "milvus" { + milvusClient, err = cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ + Address: milvusAddress, + Username: milvusConnectParam.Username, + Password: milvusConnectParam.Password, + EnableTLS: milvusConnectParam.EnableTLS, + DialConfig: milvusConnectParam.DialConfig, + }) + cancelFunc() + + if err != nil { + taskLog.Warn("fail to new target", zap.String("address", milvusAddress), zap.Error(err)) + return nil, servererror.NewClientError("fail to connect target milvus server") + } } + sourceConfig := e.config.SourceConfig etcdServerConfig := GetEtcdServerConfigFromSourceConfig(sourceConfig) metaOp, err := cdcreader.NewEtcdOp(etcdServerConfig, sourceConfig.DefaultPartitionName, config.EtcdRetryConfig{ From 21f6a96c31f4a7f6b3695e57fea6b1688dc62133 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 9 Jul 2024 16:00:16 +0900 Subject: [PATCH 016/118] multi target database --- core/reader/target_client.go | 67 ++++++++++++++++---- core/util/bigquery_client_resource.go | 82 +++++++++++++++++++++++++ core/util/common_client_resource.go | 30 +++++++++ core/util/milvus_client_resource.go | 32 ++-------- core/util/mysql_client_resource.go | 88 +++++++++++++++++++++++++++ core/util/string.go | 6 +- server/cdc_impl.go | 26 ++++---- 7 files changed, 275 insertions(+), 56 deletions(-) create mode 100644 core/util/bigquery_client_resource.go create mode 100644 core/util/common_client_resource.go create mode 100644 core/util/mysql_client_resource.go diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 8f4dde9f..df7d4f7a 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -19,7 +19,11 @@ package reader import ( + "cloud.google.com/go/bigquery" "context" + "database/sql" + "database/sql/driver" + "strings" "github.com/cockroachdb/errors" "go.uber.org/zap" @@ -35,17 +39,21 @@ import ( var _ api.TargetAPI = (*TargetClient)(nil) type TargetClient struct { - client client.Client - config TargetConfig + milvusClient client.Client + config TargetConfig + mysqlClient driver.Connector + bigqueryClient bigquery.Client } type TargetConfig struct { - Address string - Username string - Password string - APIKey string - EnableTLS bool - DialConfig util.DialConfig + TargetDBType string + Address string + Username string + Password string + ProjectId string + APIKey string + EnableTLS bool + DialConfig util.DialConfig } func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) { @@ -53,11 +61,26 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) config: config, } - _, err := targetClient.GetMilvus(ctx, "") - if err != nil { - log.Warn("fail to new target client", zap.String("address", config.Address), zap.Error(err)) - return nil, err + if strings.ToLower(config.TargetDBType) == "milvus" { + _, err := targetClient.GetMilvus(ctx, "") + if err != nil { + log.Warn("fail to new target client", zap.String("address", config.Address), zap.Error(err)) + return nil, err + } + } else if strings.ToLower(config.TargetDBType) == "mysql" { + _, err := targetClient.GetMySQL(ctx, "") + if err != nil { + log.Warn("fail to new target client", zap.String("address", config.Address), zap.Error(err)) + return nil, err + } + } else if strings.ToLower(config.TargetDBType) == "bigquery" { + _, err := targetClient.GetBigQuery(ctx, "") + if err != nil { + log.Warn("fail to new target client", zap.String("address", config.ProjectId), zap.Error(err)) + return nil, err + } } + return targetClient, nil } @@ -73,6 +96,26 @@ func (t *TargetClient) GetMilvus(ctx context.Context, databaseName string) (clie return milvusClient, nil } +func (t *TargetClient) GetMySQL(ctx context.Context, databaseName string) (*sql.DB, error) { + apiKey := t.config.APIKey + if apiKey == "" { + apiKey = util.GetAPIKey(t.config.Username, t.config.Password) + } + mysqlClient, err := util.GetMilvusClientManager().GetMySQLClient(ctx, t.config.Address, apiKey, databaseName, t.config.EnableTLS, t.config.DialConfig) + if err != nil { + return nil, err + } + return mysqlClient, nil +} + +func (t *TargetClient) GetBigQuery(ctx context.Context, databaseName string) (*bigquery.Client, error) { + bigqueryClient, err := util.GetMilvusClientManager().GetBigQueryClient(ctx, t.config.ProjectId, databaseName) + if err != nil { + return nil, err + } + return bigqueryClient, nil +} + func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, databaseName string) (*model.CollectionInfo, error) { databaseName, err := t.GetDatabaseName(ctx, collectionName, databaseName) if err != nil { diff --git a/core/util/bigquery_client_resource.go b/core/util/bigquery_client_resource.go new file mode 100644 index 00000000..0cc984ea --- /dev/null +++ b/core/util/bigquery_client_resource.go @@ -0,0 +1,82 @@ +/* + * Licensed to the LF AI & Data foundation under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * // + * http://www.apache.org/licenses/LICENSE-2.0 + * // + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "cloud.google.com/go/bigquery" + "context" + "fmt" + "reflect" + "time" + + "github.com/cockroachdb/errors" + "go.uber.org/zap" + + "github.com/milvus-io/milvus/pkg/util/resource" + + "github.com/zilliztech/milvus-cdc/core/log" +) + +const ( + BigQueryClientResourceTyp = "bigquery_client" + BigQueryClientExpireTime = 30 * time.Second + DefaultBigQueryDbName = "default" +) + +func (m *ClientResourceManager) newBigQueryClient(ctx context.Context, projectId string) resource.NewResourceFunc { + return func() (resource.Resource, error) { + + c, err := bigquery.NewClient(ctx, + projectId, + ) + if err != nil { + log.Warn("fail to new the bigquery client", zap.String("project_id", projectId), zap.Error(err)) + return nil, err + } + + res := resource.NewSimpleResource(c, BigQueryClientResourceTyp, projectId, BigQueryClientExpireTime, func() { + _ = c.Close() + }) + + return res, nil + } +} + +func (m *ClientResourceManager) GetBigQueryClient(ctx context.Context, projectId, database string) (*bigquery.Client, error) { + if database == "" { + database = DefaultBigQueryDbName + } + ctxLog := log.Ctx(ctx).With(zap.String("database", database), zap.String("project_id", projectId)) + res, err := m.manager.Get(BigQueryClientResourceTyp, + getBigQueryClientResourceName(projectId, database), + m.newBigQueryClient(ctx, projectId)) + if err != nil { + ctxLog.Error("fail to get bigquery client", zap.Error(err)) + return nil, err + } + if obj, ok := res.Get().(*bigquery.Client); ok && obj != nil { + return obj, nil + } + ctxLog.Warn("invalid resource object", zap.Any("obj", reflect.TypeOf(res.Get()))) + return nil, errors.New("invalid resource object") +} + +func getBigQueryClientResourceName(projectId, database string) string { + return fmt.Sprintf("%s:%s", projectId, database) +} diff --git a/core/util/common_client_resource.go b/core/util/common_client_resource.go new file mode 100644 index 00000000..1db9e0ab --- /dev/null +++ b/core/util/common_client_resource.go @@ -0,0 +1,30 @@ +package util + +import ( + "fmt" + "github.com/milvus-io/milvus/pkg/util/resource" + "sync" +) + +var ( + clientManager *ClientResourceManager + clientManagerOnce sync.Once +) + +type ClientResourceManager struct { + manager resource.Manager +} + +func GetMilvusClientManager() *ClientResourceManager { + clientManagerOnce.Do(func() { + manager := resource.NewManager(0, 0, nil) + clientManager = &ClientResourceManager{ + manager: manager, + } + }) + return clientManager +} + +func GetAPIKey(username, password string) string { + return fmt.Sprintf("%s:%s", username, password) +} diff --git a/core/util/milvus_client_resource.go b/core/util/milvus_client_resource.go index 9773e805..4bd48783 100644 --- a/core/util/milvus_client_resource.go +++ b/core/util/milvus_client_resource.go @@ -22,7 +22,6 @@ import ( "context" "fmt" "reflect" - "sync" "time" "github.com/cockroachdb/errors" @@ -37,29 +36,10 @@ import ( const ( MilvusClientResourceTyp = "milvus_client" MilvusClientExpireTime = 30 * time.Second - DefaultDbName = "default" + DefaultMilvusDbName = "default" ) -var ( - clientManager *MilvusClientResourceManager - clientManagerOnce sync.Once -) - -type MilvusClientResourceManager struct { - manager resource.Manager -} - -func GetMilvusClientManager() *MilvusClientResourceManager { - clientManagerOnce.Do(func() { - manager := resource.NewManager(0, 0, nil) - clientManager = &MilvusClientResourceManager{ - manager: manager, - } - }) - return clientManager -} - -func (m *MilvusClientResourceManager) newMilvusClient(ctx context.Context, address, apiKey, database string, enableTLS bool, dialConfig DialConfig) resource.NewResourceFunc { +func (m *ClientResourceManager) newMilvusClient(ctx context.Context, address, apiKey, database string, enableTLS bool, dialConfig DialConfig) resource.NewResourceFunc { return func() (resource.Resource, error) { dialOptions, err := GetDialOptions(dialConfig) if err != nil { @@ -89,9 +69,9 @@ func (m *MilvusClientResourceManager) newMilvusClient(ctx context.Context, addre } } -func (m *MilvusClientResourceManager) GetMilvusClient(ctx context.Context, address, apiKey, database string, enableTLS bool, dialConfig DialConfig) (client.Client, error) { +func (m *ClientResourceManager) GetMilvusClient(ctx context.Context, address, apiKey, database string, enableTLS bool, dialConfig DialConfig) (client.Client, error) { if database == "" { - database = DefaultDbName + database = DefaultMilvusDbName } ctxLog := log.Ctx(ctx).With(zap.String("database", database), zap.String("address", address)) res, err := m.manager.Get(MilvusClientResourceTyp, @@ -111,7 +91,3 @@ func (m *MilvusClientResourceManager) GetMilvusClient(ctx context.Context, addre func getMilvusClientResourceName(address, database string) string { return fmt.Sprintf("%s:%s", address, database) } - -func GetAPIKey(username, password string) string { - return fmt.Sprintf("%s:%s", username, password) -} diff --git a/core/util/mysql_client_resource.go b/core/util/mysql_client_resource.go new file mode 100644 index 00000000..276763cb --- /dev/null +++ b/core/util/mysql_client_resource.go @@ -0,0 +1,88 @@ +/* + * Licensed to the LF AI & Data foundation under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * // + * http://www.apache.org/licenses/LICENSE-2.0 + * // + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "context" + "database/sql" + "fmt" + "github.com/go-sql-driver/mysql" + "reflect" + "strings" + "time" + + "github.com/cockroachdb/errors" + "go.uber.org/zap" + + "github.com/milvus-io/milvus/pkg/util/resource" + + "github.com/zilliztech/milvus-cdc/core/log" +) + +const ( + MySQLClientResourceTyp = "mysql_client" + MySQLClientExpireTime = 30 * time.Second + DefaultMySQLDbName = "information_schema" +) + +func (m *ClientResourceManager) newMySQLClient(ctx context.Context, address, apiKey, database string, enableTLS bool, dialConfig DialConfig) resource.NewResourceFunc { + return func() (resource.Resource, error) { + c, err := mysql.NewConnector(&mysql.Config{ + Addr: address, + User: strings.Split(apiKey, ":")[0], + Passwd: strings.Split(apiKey, ":")[1], + DBName: database, + }) + if err != nil { + log.Warn("fail to new the mysql client", zap.String("database", database), zap.String("address", address), zap.Error(err)) + return nil, err + } + + db := sql.OpenDB(c) + res := resource.NewSimpleResource(db, MySQLClientResourceTyp, fmt.Sprintf("%s:%s", address, database), MySQLClientExpireTime, func() { + _ = db.Close() + }) + + return res, nil + } +} + +func (m *ClientResourceManager) GetMySQLClient(ctx context.Context, address, apiKey, database string, enableTLS bool, dialConfig DialConfig) (*sql.DB, error) { + if database == "" { + database = DefaultMySQLDbName + } + + ctxLog := log.Ctx(ctx).With(zap.String("database", database), zap.String("address", address)) + res, err := m.manager.Get(MySQLClientResourceTyp, + getMySQLClientResourceName(address, database), + m.newMySQLClient(ctx, address, apiKey, database, enableTLS, dialConfig)) + if err != nil { + ctxLog.Error("fail to get mysql client", zap.Error(err)) + return nil, err + } + if obj, ok := res.Get().(*sql.DB); ok && obj != nil { + return obj, nil + } + ctxLog.Warn("invalid resource object", zap.Any("obj", reflect.TypeOf(res.Get()))) + return nil, errors.New("invalid resource object") +} + +func getMySQLClientResourceName(address, database string) string { + return fmt.Sprintf("%s:%s", address, database) +} diff --git a/core/util/string.go b/core/util/string.go index 78939f72..c7879dc5 100644 --- a/core/util/string.go +++ b/core/util/string.go @@ -112,7 +112,7 @@ func GetDropInfoKey(key string) string { func GetCollectionInfoKeys(collectionName, dbName string) (string, string) { if dbName == "" { - dbName = DefaultDbName + dbName = DefaultMilvusDbName } key := fmt.Sprintf("%s_%s", dbName, collectionName) return GetCreateInfoKey(key), GetDropInfoKey(key) @@ -120,7 +120,7 @@ func GetCollectionInfoKeys(collectionName, dbName string) (string, string) { func GetPartitionInfoKeys(partitionName, collectionName, dbName string) (string, string) { if dbName == "" { - dbName = DefaultDbName + dbName = DefaultMilvusDbName } key := fmt.Sprintf("%s_%s_%s", dbName, collectionName, partitionName) return GetCreateInfoKey(key), GetDropInfoKey(key) @@ -128,7 +128,7 @@ func GetPartitionInfoKeys(partitionName, collectionName, dbName string) (string, func GetDBInfoKeys(dbName string) (string, string) { if dbName == "" { - dbName = DefaultDbName + dbName = DefaultMilvusDbName } return GetCreateInfoKey(dbName), GetDropInfoKey(dbName) } diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 123fb454..5aac22e5 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -534,20 +534,20 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err ctx := context.TODO() timeoutCtx, cancelFunc := context.WithTimeout(ctx, time.Duration(milvusConnectParam.ConnectTimeout)*time.Second) - if strings.ToLower(milvusConnectParam.TargetDBType) == "milvus" { - milvusClient, err = cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ - Address: milvusAddress, - Username: milvusConnectParam.Username, - Password: milvusConnectParam.Password, - EnableTLS: milvusConnectParam.EnableTLS, - DialConfig: milvusConnectParam.DialConfig, - }) - cancelFunc() - if err != nil { - taskLog.Warn("fail to new target", zap.String("address", milvusAddress), zap.Error(err)) - return nil, servererror.NewClientError("fail to connect target milvus server") - } + milvusClient, err = cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ + Address: milvusAddress, + Username: milvusConnectParam.Username, + Password: milvusConnectParam.Password, + EnableTLS: milvusConnectParam.EnableTLS, + DialConfig: milvusConnectParam.DialConfig, + ProjectId: milvusConnectParam.ProjectId, + }) + cancelFunc() + + if err != nil { + taskLog.Warn("fail to new target", zap.String("address", milvusAddress), zap.Error(err)) + return nil, servererror.NewClientError("fail to connect target milvus server") } sourceConfig := e.config.SourceConfig From 65f2d71423279ea59e771df1cf37146060bb44b2 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 12 Jul 2024 15:28:41 +0900 Subject: [PATCH 017/118] multi target database --- core/reader/replicate_channel_manager.go | 1 + core/reader/target_client.go | 223 +++++++++++++++++------ 2 files changed, 168 insertions(+), 56 deletions(-) diff --git a/core/reader/replicate_channel_manager.go b/core/reader/replicate_channel_manager.go index d53c7b7d..95b63b64 100644 --- a/core/reader/replicate_channel_manager.go +++ b/core/reader/replicate_channel_manager.go @@ -676,6 +676,7 @@ func (r *replicateChannelManager) forwardMsg(targetPChannel string, msg *msgstre zap.String("target_pchannel", targetPChannel), zap.Strings("channels", lo.Keys(r.channelHandlerMap))) return } + handler.forwardPackChan <- msg } diff --git a/core/reader/target_client.go b/core/reader/target_client.go index df7d4f7a..2f29d627 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -23,6 +23,8 @@ import ( "context" "database/sql" "database/sql/driver" + "github.com/milvus-io/milvus-sdk-go/v2/entity" + "google.golang.org/api/iterator" "strings" "github.com/cockroachdb/errors" @@ -122,91 +124,200 @@ func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, da log.Warn("fail to get database name", zap.Error(err)) return nil, err } - milvus, err := t.GetMilvus(ctx, databaseName) - if err != nil { - log.Warn("fail to get milvus client", zap.String("database", databaseName), zap.Error(err)) - return nil, err - } collectionInfo := &model.CollectionInfo{} - collection, err := milvus.DescribeCollection(ctx, collectionName) - if err != nil { - log.Warn("fail to describe collection", zap.Error(err)) - return nil, err + var collection *entity.Collection + if strings.ToLower(t.config.TargetDBType) == "milvus" { + milvus, err := t.GetMilvus(ctx, databaseName) + if err != nil { + log.Warn("fail to get milvus client", zap.String("database", databaseName), zap.Error(err)) + return nil, err + } + + collection, err = milvus.DescribeCollection(ctx, collectionName) + if err != nil { + log.Warn("fail to describe collection", zap.Error(err)) + return nil, err + } + + tmpCollectionInfo, err := t.GetPartitionInfo(ctx, collectionName, databaseName) + if err != nil { + log.Warn("fail to get partition info", zap.Error(err)) + return nil, err + } + + collectionInfo.Partitions = tmpCollectionInfo.Partitions + } else { + var channelName string + if strings.ToLower(t.config.TargetDBType) == "mysql" { + channelName = "mysql" + } else if strings.ToLower(t.config.TargetDBType) == "bigquery" { + channelName = "bigquery" + } + collection.PhysicalChannels = append(collection.PhysicalChannels, channelName) + collection.VirtualChannels = append(collection.VirtualChannels, channelName) } + collectionInfo.DatabaseName = databaseName collectionInfo.CollectionID = collection.ID collectionInfo.CollectionName = collectionName collectionInfo.PChannels = collection.PhysicalChannels collectionInfo.VChannels = collection.VirtualChannels - tmpCollectionInfo, err := t.GetPartitionInfo(ctx, collectionName, databaseName) - if err != nil { - log.Warn("fail to get partition info", zap.Error(err)) - return nil, err - } - collectionInfo.Partitions = tmpCollectionInfo.Partitions return collectionInfo, nil } func (t *TargetClient) GetPartitionInfo(ctx context.Context, collectionName, databaseName string) (*model.CollectionInfo, error) { - databaseName, err := t.GetDatabaseName(ctx, collectionName, databaseName) - if err != nil { - log.Warn("fail to get database name", zap.Error(err)) - return nil, err - } - milvus, err := t.GetMilvus(ctx, databaseName) - if err != nil { - log.Warn("fail to get milvus client", zap.String("database", databaseName), zap.Error(err)) - return nil, err - } + if strings.ToLower(t.config.TargetDBType) == "milvus" { + databaseName, err := t.GetDatabaseName(ctx, collectionName, databaseName) + if err != nil { + log.Warn("fail to get database name", zap.Error(err)) + return nil, err + } + milvus, err := t.GetMilvus(ctx, databaseName) + if err != nil { + log.Warn("fail to get milvus client", zap.String("database", databaseName), zap.Error(err)) + return nil, err + } - collectionInfo := &model.CollectionInfo{} - partition, err := milvus.ShowPartitions(ctx, collectionName) - if err != nil || len(partition) == 0 { - log.Warn("failed to show partitions", zap.Error(err)) - return nil, errors.New("fail to show the partitions") - } - partitionInfo := make(map[string]int64, len(partition)) - for _, e := range partition { - partitionInfo[e.Name] = e.ID + collectionInfo := &model.CollectionInfo{} + partition, err := milvus.ShowPartitions(ctx, collectionName) + if err != nil || len(partition) == 0 { + log.Warn("failed to show partitions", zap.Error(err)) + return nil, errors.New("fail to show the partitions") + } + partitionInfo := make(map[string]int64, len(partition)) + for _, e := range partition { + partitionInfo[e.Name] = e.ID + } + collectionInfo.Partitions = partitionInfo + return collectionInfo, nil } - collectionInfo.Partitions = partitionInfo - return collectionInfo, nil + + return nil, nil } func (t *TargetClient) GetDatabaseName(ctx context.Context, collectionName, databaseName string) (string, error) { if !IsDroppedObject(databaseName) { return databaseName, nil } - dbLog := log.With(zap.String("collection", collectionName), zap.String("database", databaseName)) - milvus, err := t.GetMilvus(ctx, "") - if err != nil { - dbLog.Warn("fail to get milvus client", zap.String("database", databaseName), zap.Error(err)) - return "", err - } - databaseNames, err := milvus.ListDatabases(ctx) - if err != nil { - dbLog.Warn("fail to list databases", zap.String("database", databaseName), zap.Error(err)) - return "", err - } - for _, dbName := range databaseNames { - dbMilvus, err := t.GetMilvus(ctx, dbName.Name) + if strings.ToLower(t.config.TargetDBType) == "milvus" { + dbLog := log.With(zap.String("collection", collectionName), zap.String("database", databaseName)) + milvus, err := t.GetMilvus(ctx, "") if err != nil { - dbLog.Warn("fail to get milvus client", zap.String("connect_db", dbName.Name), zap.Error(err)) + dbLog.Warn("fail to get milvus client", zap.String("database", databaseName), zap.Error(err)) return "", err } - collections, err := dbMilvus.ListCollections(ctx) + databaseNames, err := milvus.ListDatabases(ctx) if err != nil { - dbLog.Warn("fail to list collections", zap.String("connect_db", dbName.Name), zap.Error(err)) + dbLog.Warn("fail to milvus list databases", zap.String("database", databaseName), zap.Error(err)) return "", err } - for _, collection := range collections { - if collection.Name == collectionName { - return dbName.Name, nil + for _, dbName := range databaseNames { + dbMilvus, err := t.GetMilvus(ctx, dbName.Name) + if err != nil { + dbLog.Warn("fail to get milvus client", zap.String("connect_db", dbName.Name), zap.Error(err)) + return "", err + } + collections, err := dbMilvus.ListCollections(ctx) + if err != nil { + dbLog.Warn("fail to milvus list collections", zap.String("connect_db", dbName.Name), zap.Error(err)) + return "", err + } + for _, collection := range collections { + if collection.Name == collectionName { + return dbName.Name, nil + } + } + } + dbLog.Warn("not found the database", zap.Any("databases", databaseNames)) + } else if strings.ToLower(t.config.TargetDBType) == "mysql" { + dbLog := log.With(zap.String("table", collectionName), zap.String("database", databaseName)) + + var drows, trows *sql.Rows + dbMySQL, err := t.GetMySQL(ctx, "") + if err != nil { + dbLog.Warn("fail to get mysql client", zap.String("database", databaseName), zap.Error(err)) + return "", err + } + drows, err = dbMySQL.Query("show databases") + if err != nil { + dbLog.Warn("fail to mysql list databases", zap.String("database", databaseName), zap.Error(err)) + return "", err + } + + defer drows.Close() + + var dbName, tableName string + for drows.Next() { + err = drows.Scan(&dbName) + if err != nil { + dbLog.Warn("fail to mysql fetch rows", zap.String("connect_db", dbName), zap.Error(err)) + return "", err + } + + _, err = dbMySQL.Exec("use " + dbName) + if err != nil { + dbLog.Warn("fail to command mysql use db", zap.String("connect_db", dbName), zap.Error(err)) + return "", err + } + + trows, err = dbMySQL.Query("show tables") + if err != nil { + dbLog.Warn("fail to mysql list tables", zap.String("connect_db", dbName), zap.Error(err)) + return "", err + } + + for trows.Next() { + err = trows.Scan(&tableName) + if tableName == collectionName { + trows.Close() + return dbName, nil + } + } + + trows.Close() + } + dbLog.Warn("not found the database", zap.Any("databases", databaseName)) + } else if strings.ToLower(t.config.TargetDBType) == "bigquery" { + dbLog := log.With(zap.String("table", collectionName), zap.String("database", databaseName)) + + dbBigQuery, err := t.GetBigQuery(ctx, "") + if err != nil { + dbLog.Warn("fail to mysql list tables", zap.String("connect_db", databaseName), zap.Error(err)) + return "", err + } + + // 데이터셋 목록 조회 + datasets := dbBigQuery.Datasets(ctx) + for { + dataset, err := datasets.Next() + if err == iterator.Done { + break + } + if err != nil { + dbLog.Warn("failed to list datasets", zap.String("connect_db", dataset.DatasetID), zap.Error(err)) + return "", err + } + + // 각 데이터셋의 테이블 목록 조회 + tables := dataset.Tables(ctx) + for { + tableName, err := tables.Next() + if tableName.TableID == collectionName { + return dataset.DatasetID, nil + } + if err == iterator.Done { + break + } + + if err != nil { + dbLog.Warn("failed to list tables", zap.String("connect_db", dataset.DatasetID), zap.Error(err)) + return "", err + } } } } - dbLog.Warn("not found the database", zap.Any("databases", databaseNames)) + return "", util.NotFoundDatabase } From 9267dc48312d78286aa60629f4baa294e66ef1bc Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 12 Jul 2024 15:52:44 +0900 Subject: [PATCH 018/118] multi target database --- core/reader/target_client.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 2f29d627..7665bb05 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -148,14 +148,14 @@ func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, da collectionInfo.Partitions = tmpCollectionInfo.Partitions } else { - var channelName string + var channelName []string if strings.ToLower(t.config.TargetDBType) == "mysql" { - channelName = "mysql" + channelName = append(channelName, "mysql") } else if strings.ToLower(t.config.TargetDBType) == "bigquery" { - channelName = "bigquery" + channelName = append(channelName, "bigquery") } - collection.PhysicalChannels = append(collection.PhysicalChannels, channelName) - collection.VirtualChannels = append(collection.VirtualChannels, channelName) + collection.PhysicalChannels = channelName + collection.VirtualChannels = channelName } collectionInfo.DatabaseName = databaseName From 3da26a716a54f941ca6201861a209996bc84c6f8 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 12 Jul 2024 16:10:08 +0900 Subject: [PATCH 019/118] multi target database --- core/reader/target_client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 7665bb05..d798683a 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -154,8 +154,8 @@ func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, da } else if strings.ToLower(t.config.TargetDBType) == "bigquery" { channelName = append(channelName, "bigquery") } - collection.PhysicalChannels = channelName - collection.VirtualChannels = channelName + // collection.PhysicalChannels = channelName + // collection.VirtualChannels = channelName } collectionInfo.DatabaseName = databaseName From d1cc39f5ce7fc8a679c80dd3af841b57de999bd9 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 12 Jul 2024 16:21:25 +0900 Subject: [PATCH 020/118] multi target database --- core/reader/target_client.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index d798683a..5d0f7cde 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -148,14 +148,15 @@ func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, da collectionInfo.Partitions = tmpCollectionInfo.Partitions } else { - var channelName []string + collection = &entity.Collection{} + var channelName string if strings.ToLower(t.config.TargetDBType) == "mysql" { - channelName = append(channelName, "mysql") + channelName = "mysql" } else if strings.ToLower(t.config.TargetDBType) == "bigquery" { - channelName = append(channelName, "bigquery") + channelName = "bigquery" } - // collection.PhysicalChannels = channelName - // collection.VirtualChannels = channelName + collection.PhysicalChannels = append(collection.PhysicalChannels, channelName) + collection.VirtualChannels = append(collection.VirtualChannels, channelName) } collectionInfo.DatabaseName = databaseName From 83596fc9d964fbdd729c18c8230c97334227d19e Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 12 Jul 2024 18:32:06 +0900 Subject: [PATCH 021/118] multi target database --- core/reader/replicate_channel_manager.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/core/reader/replicate_channel_manager.go b/core/reader/replicate_channel_manager.go index 95b63b64..63aa8b8a 100644 --- a/core/reader/replicate_channel_manager.go +++ b/core/reader/replicate_channel_manager.go @@ -279,9 +279,17 @@ func (r *replicateChannelManager) StartReadCollection(ctx context.Context, info var successChannels []string var channelHandlers []*replicateChannelHandler + toPhysicalChannel := func(vChannel string) string { + if strings.ToLower(vChannel) != "mysql" && strings.ToLower(vChannel) != "bigquery" { + return funcutil.ToPhysicalChannel(vChannel) + } + + return strings.ToLower(vChannel) + } + err = ForeachChannel(info.VirtualChannelNames, targetInfo.VChannels, func(sourceVChannel, targetVChannel string) error { - sourcePChannel := funcutil.ToPhysicalChannel(sourceVChannel) - targetPChannel := funcutil.ToPhysicalChannel(targetVChannel) + sourcePChannel := toPhysicalChannel(sourceVChannel) + targetPChannel := toPhysicalChannel(targetVChannel) channelHandler, err := r.startReadChannel(&model.SourceCollectionInfo{ PChannelName: sourcePChannel, VChannelName: sourceVChannel, @@ -558,11 +566,13 @@ func (r *replicateChannelManager) startReadChannel(sourceInfo *model.SourceColle hasReplicateForTargetChannel := false for _, handler := range r.channelHandlerMap { handler.recordLock.RLock() - if handler.targetPChannel == targetInfo.PChannel { + + if handler.targetPChannel == targetInfo.PChannel && (targetInfo.PChannel != "mysql" && targetInfo.PChannel != "bigquery") { hasReplicateForTargetChannel = true } handler.recordLock.RUnlock() } + if hasReplicateForTargetChannel { channelLog.Info("channel already has replicate for target channel", zap.String("channel_name", sourceInfo.PChannelName)) r.waitChannel(sourceInfo, targetInfo, channelHandler) From 6a0ec216cdf42d1f331c9cf798185657ea83f4e4 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 12 Jul 2024 19:11:06 +0900 Subject: [PATCH 022/118] multi target database --- core/reader/replicate_channel_manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/reader/replicate_channel_manager.go b/core/reader/replicate_channel_manager.go index 63aa8b8a..f814d02e 100644 --- a/core/reader/replicate_channel_manager.go +++ b/core/reader/replicate_channel_manager.go @@ -287,6 +287,7 @@ func (r *replicateChannelManager) StartReadCollection(ctx context.Context, info return strings.ToLower(vChannel) } + log.Info("target information", zap.Any("target_info", targetInfo.VChannels), zap.Any("collection_id", info.ID)) err = ForeachChannel(info.VirtualChannelNames, targetInfo.VChannels, func(sourceVChannel, targetVChannel string) error { sourcePChannel := toPhysicalChannel(sourceVChannel) targetPChannel := toPhysicalChannel(targetVChannel) From a875f15862833215c5544d56b7a2870623b63a17 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 12 Jul 2024 19:31:07 +0900 Subject: [PATCH 023/118] multi target database --- core/reader/target_client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 5d0f7cde..6f17a0f0 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -257,6 +257,7 @@ func (t *TargetClient) GetDatabaseName(ctx context.Context, collectionName, data return "", err } + dbLog.Info("databaseName", zap.String("databaseName", dbName)) _, err = dbMySQL.Exec("use " + dbName) if err != nil { dbLog.Warn("fail to command mysql use db", zap.String("connect_db", dbName), zap.Error(err)) @@ -271,6 +272,7 @@ func (t *TargetClient) GetDatabaseName(ctx context.Context, collectionName, data for trows.Next() { err = trows.Scan(&tableName) + dbLog.Info("tableName", zap.String("tableName", tableName)) if tableName == collectionName { trows.Close() return dbName, nil From 3abe66fa9e42f1ce82ad2f65449aa0bc50d91cf5 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 12 Jul 2024 19:43:31 +0900 Subject: [PATCH 024/118] multi target database --- core/reader/target_client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 6f17a0f0..a14827c4 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -199,7 +199,9 @@ func (t *TargetClient) GetPartitionInfo(ctx context.Context, collectionName, dat } func (t *TargetClient) GetDatabaseName(ctx context.Context, collectionName, databaseName string) (string, error) { + log.Info("GetDatabaseName", zap.String("collectionName", collectionName), zap.String("databaseName", databaseName)) if !IsDroppedObject(databaseName) { + log.Warn("database name is not dropped", zap.String("databaseName", databaseName)) return databaseName, nil } if strings.ToLower(t.config.TargetDBType) == "milvus" { From 7849e8eb6df9ee1dfe552537cd25e5759b86504e Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 12 Jul 2024 19:49:28 +0900 Subject: [PATCH 025/118] multi target database --- core/reader/target_client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index a14827c4..5461422a 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -165,6 +165,8 @@ func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, da collectionInfo.PChannels = collection.PhysicalChannels collectionInfo.VChannels = collection.VirtualChannels + log.Info("GetCollectionInfo", zap.Any("collectionInfo", collectionInfo)) + return collectionInfo, nil } From 165b82ebe4840691403aa1f32919916804b21e96 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 16 Jul 2024 11:42:31 +0900 Subject: [PATCH 026/118] multi target database --- core/api/replicate_manager.go | 8 ++-- core/reader/collection_reader.go | 8 ++-- core/reader/replicate_channel_manager.go | 6 +-- core/reader/target_client.go | 53 ++++++++++++------------ server/cdc_impl.go | 15 +++---- 5 files changed, 46 insertions(+), 44 deletions(-) diff --git a/core/api/replicate_manager.go b/core/api/replicate_manager.go index 8a2c5153..187babf9 100644 --- a/core/api/replicate_manager.go +++ b/core/api/replicate_manager.go @@ -36,7 +36,7 @@ type ChannelManager interface { AddDroppedCollection(ids []int64) AddDroppedPartition(ids []int64) - StartReadCollection(ctx context.Context, info *pb.CollectionInfo, seekPositions []*msgpb.MsgPosition) error + StartReadCollection(ctx context.Context, info *pb.CollectionInfo, targetDBType string, seekPositions []*msgpb.MsgPosition) error StopReadCollection(ctx context.Context, info *pb.CollectionInfo) error AddPartition(ctx context.Context, collectionInfo *pb.CollectionInfo, partitionInfo *pb.PartitionInfo) error @@ -46,7 +46,7 @@ type ChannelManager interface { } type TargetAPI interface { - GetCollectionInfo(ctx context.Context, collectionName, databaseName string) (*model.CollectionInfo, error) + GetCollectionInfo(ctx context.Context, targetDBType, collectionName, databaseName string) (*model.CollectionInfo, error) GetPartitionInfo(ctx context.Context, collectionName, databaseName string) (*model.CollectionInfo, error) GetDatabaseName(ctx context.Context, collectionName, databaseName string) (string, error) } @@ -102,7 +102,7 @@ func (d *DefaultChannelManager) AddDroppedPartition(ids []int64) { log.Warn("AddDroppedPartition is not implemented, please check it") } -func (d *DefaultChannelManager) StartReadCollection(ctx context.Context, info *pb.CollectionInfo, seekPositions []*msgpb.MsgPosition) error { +func (d *DefaultChannelManager) StartReadCollection(ctx context.Context, info *pb.CollectionInfo, targetDBType string, seekPositions []*msgpb.MsgPosition) error { log.Warn("StartReadCollection is not implemented, please check it") return nil } @@ -136,7 +136,7 @@ type DefaultTargetAPI struct{} var _ TargetAPI = (*DefaultTargetAPI)(nil) -func (d *DefaultTargetAPI) GetCollectionInfo(ctx context.Context, collectionName, databaseName string) (*model.CollectionInfo, error) { +func (d *DefaultTargetAPI) GetCollectionInfo(ctx context.Context, targetDBType, collectionName, databaseName string) (*model.CollectionInfo, error) { log.Warn("GetCollectionInfo is not implemented, please check it") return nil, nil } diff --git a/core/reader/collection_reader.go b/core/reader/collection_reader.go index 268375f2..d57545c2 100644 --- a/core/reader/collection_reader.go +++ b/core/reader/collection_reader.go @@ -67,9 +67,10 @@ type CollectionReader struct { quitOnce sync.Once retryOptions []retry.Option + targetDBType string } -func NewCollectionReader(id string, +func NewCollectionReader(id string, targetDBType string, channelManager api.ChannelManager, metaOp api.MetaOp, seekPosition map[string]*msgpb.MsgPosition, shouldReadFunc ShouldReadFunc, @@ -83,6 +84,7 @@ func NewCollectionReader(id string, shouldReadFunc: shouldReadFunc, errChan: make(chan error), retryOptions: util.GetRetryOptions(readerConfig.Retry), + targetDBType: targetDBType, } return reader, nil } @@ -114,7 +116,7 @@ func (reader *CollectionReader) StartRead(ctx context.Context) { Timestamp: info.CreateTime, }) } - if err := reader.channelManager.StartReadCollection(ctx, info, startPositions); err != nil { + if err := reader.channelManager.StartReadCollection(ctx, info, reader.targetDBType, startPositions); err != nil { collectionLog.Warn("fail to start to replicate the collection data in the watch process", zap.Any("info", info), zap.Error(err)) reader.sendError(err) } @@ -226,7 +228,7 @@ func (reader *CollectionReader) StartRead(ctx context.Context) { zap.String("name", info.Schema.Name), zap.Int64("collection_id", info.ID), zap.String("state", info.State.String())) - if err := reader.channelManager.StartReadCollection(ctx, info, seekPositions); err != nil { + if err := reader.channelManager.StartReadCollection(ctx, info, reader.targetDBType, seekPositions); err != nil { readerLog.Warn("fail to start to replicate the collection data", zap.Any("collection", info), zap.Error(err)) reader.sendError(err) } diff --git a/core/reader/replicate_channel_manager.go b/core/reader/replicate_channel_manager.go index f814d02e..a9e0145d 100644 --- a/core/reader/replicate_channel_manager.go +++ b/core/reader/replicate_channel_manager.go @@ -159,7 +159,7 @@ func (r *replicateChannelManager) AddDroppedPartition(ids []int64) { log.Info("has removed dropped partitions", zap.Int64s("ids", ids)) } -func (r *replicateChannelManager) StartReadCollection(ctx context.Context, info *pb.CollectionInfo, seekPositions []*msgpb.MsgPosition) error { +func (r *replicateChannelManager) StartReadCollection(ctx context.Context, info *pb.CollectionInfo, targetDBType string, seekPositions []*msgpb.MsgPosition) error { r.addCollectionLock.Lock() *r.addCollectionCnt++ r.addCollectionLock.Unlock() @@ -174,7 +174,7 @@ func (r *replicateChannelManager) StartReadCollection(ctx context.Context, info var err error retryErr := retry.Do(ctx, func() error { - _, err = r.targetClient.GetCollectionInfo(ctx, info.Schema.GetName(), sourceDBInfo.Name) + _, err = r.targetClient.GetCollectionInfo(ctx, targetDBType, info.Schema.GetName(), sourceDBInfo.Name) if err != nil && !IsCollectionNotFoundError(err) && !IsDatabaseNotFoundError(err) { return err @@ -226,7 +226,7 @@ func (r *replicateChannelManager) StartReadCollection(ctx context.Context, info var targetInfo *model.CollectionInfo err = retry.Do(ctx, func() error { - targetInfo, err = r.targetClient.GetCollectionInfo(ctx, info.Schema.Name, sourceDBInfo.Name) + targetInfo, err = r.targetClient.GetCollectionInfo(ctx, targetDBType, info.Schema.Name, sourceDBInfo.Name) return err }, r.startReadRetryOptions...) if err != nil { diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 5461422a..6215ad17 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -118,7 +118,7 @@ func (t *TargetClient) GetBigQuery(ctx context.Context, databaseName string) (*b return bigqueryClient, nil } -func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, databaseName string) (*model.CollectionInfo, error) { +func (t *TargetClient) GetCollectionInfo(ctx context.Context, targetDBType, collectionName, databaseName string) (*model.CollectionInfo, error) { databaseName, err := t.GetDatabaseName(ctx, collectionName, databaseName) if err != nil { log.Warn("fail to get database name", zap.Error(err)) @@ -127,7 +127,8 @@ func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, da collectionInfo := &model.CollectionInfo{} var collection *entity.Collection - if strings.ToLower(t.config.TargetDBType) == "milvus" { + + if strings.ToLower(targetDBType) == "milvus" { milvus, err := t.GetMilvus(ctx, databaseName) if err != nil { log.Warn("fail to get milvus client", zap.String("database", databaseName), zap.Error(err)) @@ -150,9 +151,9 @@ func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, da } else { collection = &entity.Collection{} var channelName string - if strings.ToLower(t.config.TargetDBType) == "mysql" { + if strings.ToLower(targetDBType) == "mysql" { channelName = "mysql" - } else if strings.ToLower(t.config.TargetDBType) == "bigquery" { + } else if strings.ToLower(targetDBType) == "bigquery" { channelName = "bigquery" } collection.PhysicalChannels = append(collection.PhysicalChannels, channelName) @@ -171,31 +172,29 @@ func (t *TargetClient) GetCollectionInfo(ctx context.Context, collectionName, da } func (t *TargetClient) GetPartitionInfo(ctx context.Context, collectionName, databaseName string) (*model.CollectionInfo, error) { - if strings.ToLower(t.config.TargetDBType) == "milvus" { - databaseName, err := t.GetDatabaseName(ctx, collectionName, databaseName) - if err != nil { - log.Warn("fail to get database name", zap.Error(err)) - return nil, err - } - milvus, err := t.GetMilvus(ctx, databaseName) - if err != nil { - log.Warn("fail to get milvus client", zap.String("database", databaseName), zap.Error(err)) - return nil, err - } + databaseName, err := t.GetDatabaseName(ctx, collectionName, databaseName) + if err != nil { + log.Warn("fail to get database name", zap.Error(err)) + return nil, err + } + milvus, err := t.GetMilvus(ctx, databaseName) + if err != nil { + log.Warn("fail to get milvus client", zap.String("database", databaseName), zap.Error(err)) + return nil, err + } - collectionInfo := &model.CollectionInfo{} - partition, err := milvus.ShowPartitions(ctx, collectionName) - if err != nil || len(partition) == 0 { - log.Warn("failed to show partitions", zap.Error(err)) - return nil, errors.New("fail to show the partitions") - } - partitionInfo := make(map[string]int64, len(partition)) - for _, e := range partition { - partitionInfo[e.Name] = e.ID - } - collectionInfo.Partitions = partitionInfo - return collectionInfo, nil + collectionInfo := &model.CollectionInfo{} + partition, err := milvus.ShowPartitions(ctx, collectionName) + if err != nil || len(partition) == 0 { + log.Warn("failed to show partitions", zap.Error(err)) + return nil, errors.New("fail to show the partitions") + } + partitionInfo := make(map[string]int64, len(partition)) + for _, e := range partition { + partitionInfo[e.Name] = e.ID } + collectionInfo.Partitions = partitionInfo + return collectionInfo, nil return nil, nil } diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 5aac22e5..ceb1a198 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -467,7 +467,7 @@ func (e *MetaCDC) startInternal(info *meta.TaskInfo, ignoreUpdateState bool) err } } } - collectionReader, err := cdcreader.NewCollectionReader(info.TaskID, + collectionReader, err := cdcreader.NewCollectionReader(info.TaskID, info.MilvusConnectParam.TargetDBType, replicateEntity.channelManager, replicateEntity.metaOp, channelSeekPosition, GetShouldReadFunc(info), config.ReaderConfig{ @@ -536,12 +536,13 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err timeoutCtx, cancelFunc := context.WithTimeout(ctx, time.Duration(milvusConnectParam.ConnectTimeout)*time.Second) milvusClient, err = cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ - Address: milvusAddress, - Username: milvusConnectParam.Username, - Password: milvusConnectParam.Password, - EnableTLS: milvusConnectParam.EnableTLS, - DialConfig: milvusConnectParam.DialConfig, - ProjectId: milvusConnectParam.ProjectId, + Address: milvusAddress, + Username: milvusConnectParam.Username, + Password: milvusConnectParam.Password, + EnableTLS: milvusConnectParam.EnableTLS, + DialConfig: milvusConnectParam.DialConfig, + ProjectId: milvusConnectParam.ProjectId, + TargetDBType: info.MilvusConnectParam.TargetDBType, }) cancelFunc() From 92a027dc836d4930754a4add3c3920f9a523667d Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 16 Jul 2024 15:08:15 +0900 Subject: [PATCH 027/118] multi target database --- server/model/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/model/common.go b/server/model/common.go index 6f6b8593..8938e42e 100644 --- a/server/model/common.go +++ b/server/model/common.go @@ -22,7 +22,7 @@ import "github.com/zilliztech/milvus-cdc/core/util" //go:generate easytags $GOFILE json,mapstructure type MilvusConnectParam struct { - TargetDBType string `json:"target_db_type" mapstructure:"target_db_type,omitempty"` + TargetDBType string `json:"target_db_type" mapstructure:"target_db_type"` Host string `json:"host" mapstructure:"host"` Port int `json:"port" mapstructure:"port"` Username string `json:"username,omitempty" mapstructure:"username,omitempty"` From 805edcc176a03c51e7267f155bd130e0abfe7f37 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 16 Jul 2024 19:54:19 +0900 Subject: [PATCH 028/118] multi target database --- server/cdc_impl.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index ceb1a198..14ed8c45 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -551,6 +551,8 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err return nil, servererror.NewClientError("fail to connect target milvus server") } + taskLog.Info("milvusClient", zap.Any("milvusClient", milvusClient)) + sourceConfig := e.config.SourceConfig etcdServerConfig := GetEtcdServerConfigFromSourceConfig(sourceConfig) metaOp, err := cdcreader.NewEtcdOp(etcdServerConfig, sourceConfig.DefaultPartitionName, config.EtcdRetryConfig{ From 13acc9c582107959500bc1fb06cd1f4a4d0e0f15 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 16 Jul 2024 20:03:16 +0900 Subject: [PATCH 029/118] multi target database --- server/cdc_impl.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 14ed8c45..329e3f36 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -551,6 +551,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err return nil, servererror.NewClientError("fail to connect target milvus server") } + taskLog.Info("taskInfo", zap.Any("taskInfo", milvusClient)) taskLog.Info("milvusClient", zap.Any("milvusClient", milvusClient)) sourceConfig := e.config.SourceConfig From 8fc7e0a038b942d5f68e16d5df64c43a5451cd73 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 17 Jul 2024 19:22:30 +0900 Subject: [PATCH 030/118] multi target database --- server/cdc_impl.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 329e3f36..4215af36 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -248,6 +248,9 @@ func (e *MetaCDC) Create(req *request.CreateRequest) (resp *request.CreateRespon WriterCacheConfig: req.BufferConfig, State: meta.TaskStateInitial, } + + log.Info("create task", zap.Any("task_info", info)) + if len(req.Positions) != 0 { positions := make(map[string]*meta.PositionInfo, len(req.Positions)) for s, s2 := range req.Positions { From 0510c3bd5c0b4cba9f75203f39709e43f6cf7b4f Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 17 Jul 2024 19:40:08 +0900 Subject: [PATCH 031/118] multi target database --- server/cdc_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 4215af36..6d0aff20 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -554,7 +554,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err return nil, servererror.NewClientError("fail to connect target milvus server") } - taskLog.Info("taskInfo", zap.Any("taskInfo", milvusClient)) + taskLog.Info("taskInfo", zap.Any("taskInfo", info)) taskLog.Info("milvusClient", zap.Any("milvusClient", milvusClient)) sourceConfig := e.config.SourceConfig From 719b236c6ef1e96440f4fc2d0940ae9f22854189 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 17 Jul 2024 21:46:26 +0900 Subject: [PATCH 032/118] multi target database --- server/cdc_impl.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 6d0aff20..322576c7 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -555,6 +555,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err } taskLog.Info("taskInfo", zap.Any("taskInfo", info)) + taskLog.Info("target db type", zap.Any("target db type", info.MilvusConnectParam.TargetDBType)) taskLog.Info("milvusClient", zap.Any("milvusClient", milvusClient)) sourceConfig := e.config.SourceConfig From 60215878c4b92acfbae789e4b17a67f226e5a55b Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 17 Jul 2024 21:58:17 +0900 Subject: [PATCH 033/118] multi target database --- core/reader/target_client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 6215ad17..3c82bed8 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -83,6 +83,7 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) } } + log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", config)) return targetClient, nil } From 4d3088c10f08e0674f11b3ed26d2cb1db7e2b641 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 17 Jul 2024 22:13:29 +0900 Subject: [PATCH 034/118] multi target database --- core/reader/target_client.go | 2 +- server/cdc_impl.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 3c82bed8..05ff6a7d 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -83,7 +83,7 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) } } - log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", config)) + log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", targetClient)) return targetClient, nil } diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 322576c7..3e310d50 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -532,13 +532,10 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err milvusConnectParam := info.MilvusConnectParam milvusAddress := fmt.Sprintf("%s:%d", milvusConnectParam.Host, milvusConnectParam.Port) - var milvusClient api.TargetAPI - var err error - ctx := context.TODO() timeoutCtx, cancelFunc := context.WithTimeout(ctx, time.Duration(milvusConnectParam.ConnectTimeout)*time.Second) - milvusClient, err = cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ + milvusClient, err := cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ Address: milvusAddress, Username: milvusConnectParam.Username, Password: milvusConnectParam.Password, From 390fc53b5b7fad2f95f3f226140e101968c43bf8 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 17 Jul 2024 23:25:36 +0900 Subject: [PATCH 035/118] multi target database --- core/reader/target_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 05ff6a7d..47f96271 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -83,7 +83,7 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) } } - log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", targetClient)) + log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", *targetClient)) return targetClient, nil } From 372fb08b5a06d59c019e2dbf54a602800f626d1e Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 17 Jul 2024 23:38:18 +0900 Subject: [PATCH 036/118] multi target database --- core/reader/target_client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 47f96271..f1facaba 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -63,6 +63,7 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) config: config, } + log.Info("new target", zap.Any("targetClient", targetClient)) if strings.ToLower(config.TargetDBType) == "milvus" { _, err := targetClient.GetMilvus(ctx, "") if err != nil { @@ -83,7 +84,7 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) } } - log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", *targetClient)) + log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", targetClient)) return targetClient, nil } From 4275de2a09ea2ca1cafaecbf8ed46b3a2004e645 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 17 Jul 2024 23:51:20 +0900 Subject: [PATCH 037/118] multi target database --- core/reader/target_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index f1facaba..eccc86df 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -63,7 +63,7 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) config: config, } - log.Info("new target", zap.Any("targetClient", targetClient)) + log.Info("new target", zap.Any("targetClient", *targetClient)) if strings.ToLower(config.TargetDBType) == "milvus" { _, err := targetClient.GetMilvus(ctx, "") if err != nil { From b8c497a7c8014cd1c9ab7992d404bb031c9e5e2a Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 18 Jul 2024 01:36:00 +0900 Subject: [PATCH 038/118] multi target database --- core/reader/target_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index eccc86df..4e75ef7d 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -63,7 +63,7 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) config: config, } - log.Info("new target", zap.Any("targetClient", *targetClient)) + log.Info("new target", zap.Any("targetClient", targetClient.config)) if strings.ToLower(config.TargetDBType) == "milvus" { _, err := targetClient.GetMilvus(ctx, "") if err != nil { From 2e7b93213307e2d2665a7154f57b37c6de4872ae Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 18 Jul 2024 15:10:19 +0900 Subject: [PATCH 039/118] multi target database --- core/reader/target_client.go | 9 ++-- server/cdc_impl.go | 89 ++++++++++++++++++++++++++---------- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 4e75ef7d..d20625d0 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -63,20 +63,19 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) config: config, } - log.Info("new target", zap.Any("targetClient", targetClient.config)) - if strings.ToLower(config.TargetDBType) == "milvus" { + if strings.ToLower(targetClient.config.TargetDBType) == "milvus" { _, err := targetClient.GetMilvus(ctx, "") if err != nil { log.Warn("fail to new target client", zap.String("address", config.Address), zap.Error(err)) return nil, err } - } else if strings.ToLower(config.TargetDBType) == "mysql" { + } else if strings.ToLower(targetClient.config.TargetDBType) == "mysql" { _, err := targetClient.GetMySQL(ctx, "") if err != nil { log.Warn("fail to new target client", zap.String("address", config.Address), zap.Error(err)) return nil, err } - } else if strings.ToLower(config.TargetDBType) == "bigquery" { + } else if strings.ToLower(targetClient.config.TargetDBType) == "bigquery" { _, err := targetClient.GetBigQuery(ctx, "") if err != nil { log.Warn("fail to new target client", zap.String("address", config.ProjectId), zap.Error(err)) @@ -84,7 +83,7 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) } } - log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", targetClient)) + log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", targetClient.config)) return targetClient, nil } diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 3e310d50..5941ef61 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -596,7 +596,6 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err return nil, servererror.NewClientError("fail to create replicate channel manager") } targetConfig := milvusConnectParam - var writerObj api.Writer if strings.ToLower(targetConfig.TargetDBType) == "milvus" { dataHandler, err := cdcwriter.NewMilvusDataHandler( @@ -610,10 +609,31 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err taskLog.Warn("fail to new the milvus data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } - writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ + writerObj := cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ MessageBufferSize: bufferSize, Retry: e.config.Retry, }, metaOp.GetAllDroppedObj()) + + e.replicateEntityMap.Lock() + defer e.replicateEntityMap.Unlock() + entity, ok := e.replicateEntityMap.data[milvusAddress] + if !ok { + replicateCtx, cancelReplicateFunc := context.WithCancel(ctx) + channelManager.SetCtx(replicateCtx) + entity = &ReplicateEntity{ + targetClient: milvusClient, + channelManager: channelManager, + metaOp: metaOp, + writerObj: writerObj, + quitFunc: cancelReplicateFunc, + mqDispatcher: msgDispatcherClient, + mqTTDispatcher: msgTTDispatcherClient, + } + e.replicateEntityMap.data[milvusAddress] = entity + e.startReplicateAPIEvent(replicateCtx, info, entity) + e.startReplicateDMLChannel(replicateCtx, info, entity) + } + return entity, nil } else if strings.ToLower(targetConfig.TargetDBType) == "bigquery" { dataHandler, err := cdcwriter.NewBigQueryDataHandler( cdcwriter.ProjectOption(targetConfig.ProjectId), @@ -623,10 +643,31 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err taskLog.Warn("fail to new the bigquery data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } - writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ + writerObj := cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ MessageBufferSize: bufferSize, Retry: e.config.Retry, }, metaOp.GetAllDroppedObj()) + + e.replicateEntityMap.Lock() + defer e.replicateEntityMap.Unlock() + entity, ok := e.replicateEntityMap.data[milvusAddress] + if !ok { + replicateCtx, cancelReplicateFunc := context.WithCancel(ctx) + channelManager.SetCtx(replicateCtx) + entity = &ReplicateEntity{ + targetClient: milvusClient, + channelManager: channelManager, + metaOp: metaOp, + writerObj: writerObj, + quitFunc: cancelReplicateFunc, + mqDispatcher: msgDispatcherClient, + mqTTDispatcher: msgTTDispatcherClient, + } + e.replicateEntityMap.data[milvusAddress] = entity + e.startReplicateAPIEvent(replicateCtx, info, entity) + e.startReplicateDMLChannel(replicateCtx, info, entity) + } + return entity, nil } else if strings.ToLower(targetConfig.TargetDBType) == "mysql" { dataHandler, err := cdcwriter.NewMySQLDataHandler( cdcwriter.MySQLAddressOption(fmt.Sprintf("%s:%d", targetConfig.Host, targetConfig.Port)), @@ -637,32 +678,34 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err taskLog.Warn("fail to new the mysql data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } - writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ + writerObj := cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ MessageBufferSize: bufferSize, Retry: e.config.Retry, }, metaOp.GetAllDroppedObj()) - } - e.replicateEntityMap.Lock() - defer e.replicateEntityMap.Unlock() - entity, ok := e.replicateEntityMap.data[milvusAddress] - if !ok { - replicateCtx, cancelReplicateFunc := context.WithCancel(ctx) - channelManager.SetCtx(replicateCtx) - entity = &ReplicateEntity{ - targetClient: milvusClient, - channelManager: channelManager, - metaOp: metaOp, - writerObj: writerObj, - quitFunc: cancelReplicateFunc, - mqDispatcher: msgDispatcherClient, - mqTTDispatcher: msgTTDispatcherClient, + e.replicateEntityMap.Lock() + defer e.replicateEntityMap.Unlock() + entity, ok := e.replicateEntityMap.data[milvusAddress] + if !ok { + replicateCtx, cancelReplicateFunc := context.WithCancel(ctx) + channelManager.SetCtx(replicateCtx) + entity = &ReplicateEntity{ + targetClient: milvusClient, + channelManager: channelManager, + metaOp: metaOp, + writerObj: writerObj, + quitFunc: cancelReplicateFunc, + mqDispatcher: msgDispatcherClient, + mqTTDispatcher: msgTTDispatcherClient, + } + e.replicateEntityMap.data[milvusAddress] = entity + e.startReplicateAPIEvent(replicateCtx, info, entity) + e.startReplicateDMLChannel(replicateCtx, info, entity) } - e.replicateEntityMap.data[milvusAddress] = entity - e.startReplicateAPIEvent(replicateCtx, info, entity) - e.startReplicateDMLChannel(replicateCtx, info, entity) + return entity, nil } - return entity, nil + + return nil, nil } func (e *MetaCDC) startReplicateAPIEvent(replicateCtx context.Context, info *meta.TaskInfo, entity *ReplicateEntity) { From 51772468df35ce3682126d8bc47bdeffb311db9c Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 19 Jul 2024 15:41:49 +0900 Subject: [PATCH 040/118] multi target database --- core/reader/replicate_channel_manager.go | 26 +++++-- core/reader/target_client.go | 17 ++--- server/cdc_impl.go | 89 ++++++------------------ 3 files changed, 47 insertions(+), 85 deletions(-) diff --git a/core/reader/replicate_channel_manager.go b/core/reader/replicate_channel_manager.go index a9e0145d..241c15e0 100644 --- a/core/reader/replicate_channel_manager.go +++ b/core/reader/replicate_channel_manager.go @@ -20,6 +20,7 @@ package reader import ( "context" + "fmt" "io" "math" "sort" @@ -287,8 +288,7 @@ func (r *replicateChannelManager) StartReadCollection(ctx context.Context, info return strings.ToLower(vChannel) } - log.Info("target information", zap.Any("target_info", targetInfo.VChannels), zap.Any("collection_id", info.ID)) - err = ForeachChannel(info.VirtualChannelNames, targetInfo.VChannels, func(sourceVChannel, targetVChannel string) error { + err = ForeachChannel(targetDBType, info.VirtualChannelNames, targetInfo.VChannels, func(sourceVChannel, targetVChannel string) error { sourcePChannel := toPhysicalChannel(sourceVChannel) targetPChannel := toPhysicalChannel(targetVChannel) channelHandler, err := r.startReadChannel(&model.SourceCollectionInfo{ @@ -344,12 +344,25 @@ func GetVChannelByPChannel(pChannel string, vChannels []string) string { return "" } -func ForeachChannel(sourcePChannels, targetPChannels []string, f func(sourcePChannel, targetPChannel string) error) error { - if len(sourcePChannels) != len(targetPChannels) { - return errors.New("the lengths of source and target channels are not equal") +func ForeachChannel(targetDBType string, sourcePChannels, targetPChannels []string, f func(sourcePChannel, targetPChannel string) error) error { + if targetDBType == "milvus" { + if len(sourcePChannels) != len(targetPChannels) { + return errors.New("the lengths of source and target channels are not equal") + } } + sources := make([]string, len(sourcePChannels)) - targets := make([]string, len(targetPChannels)) + var targets []string + + if targetDBType == "milvus" { + targets = make([]string, len(targetPChannels)) + } else { + targets = make([]string, len(sourcePChannels)) + for i := 0; i < len(sourcePChannels); i++ { + targetPChannels = append(targetPChannels, fmt.Sprintf("%s-dml_%d_v0", targetDBType, i)) + } + } + copy(sources, sourcePChannels) copy(targets, targetPChannels) sort.Strings(sources) @@ -360,6 +373,7 @@ func ForeachChannel(sourcePChannels, targetPChannels []string, f func(sourcePCha return err } } + return nil } diff --git a/core/reader/target_client.go b/core/reader/target_client.go index d20625d0..2ce89dfb 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -149,23 +149,14 @@ func (t *TargetClient) GetCollectionInfo(ctx context.Context, targetDBType, coll } collectionInfo.Partitions = tmpCollectionInfo.Partitions - } else { - collection = &entity.Collection{} - var channelName string - if strings.ToLower(targetDBType) == "mysql" { - channelName = "mysql" - } else if strings.ToLower(targetDBType) == "bigquery" { - channelName = "bigquery" - } - collection.PhysicalChannels = append(collection.PhysicalChannels, channelName) - collection.VirtualChannels = append(collection.VirtualChannels, channelName) + + collectionInfo.CollectionID = collection.ID + collectionInfo.PChannels = collection.PhysicalChannels + collectionInfo.VChannels = collection.VirtualChannels } collectionInfo.DatabaseName = databaseName - collectionInfo.CollectionID = collection.ID collectionInfo.CollectionName = collectionName - collectionInfo.PChannels = collection.PhysicalChannels - collectionInfo.VChannels = collection.VirtualChannels log.Info("GetCollectionInfo", zap.Any("collectionInfo", collectionInfo)) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 5941ef61..3e310d50 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -596,6 +596,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err return nil, servererror.NewClientError("fail to create replicate channel manager") } targetConfig := milvusConnectParam + var writerObj api.Writer if strings.ToLower(targetConfig.TargetDBType) == "milvus" { dataHandler, err := cdcwriter.NewMilvusDataHandler( @@ -609,31 +610,10 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err taskLog.Warn("fail to new the milvus data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } - writerObj := cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ + writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ MessageBufferSize: bufferSize, Retry: e.config.Retry, }, metaOp.GetAllDroppedObj()) - - e.replicateEntityMap.Lock() - defer e.replicateEntityMap.Unlock() - entity, ok := e.replicateEntityMap.data[milvusAddress] - if !ok { - replicateCtx, cancelReplicateFunc := context.WithCancel(ctx) - channelManager.SetCtx(replicateCtx) - entity = &ReplicateEntity{ - targetClient: milvusClient, - channelManager: channelManager, - metaOp: metaOp, - writerObj: writerObj, - quitFunc: cancelReplicateFunc, - mqDispatcher: msgDispatcherClient, - mqTTDispatcher: msgTTDispatcherClient, - } - e.replicateEntityMap.data[milvusAddress] = entity - e.startReplicateAPIEvent(replicateCtx, info, entity) - e.startReplicateDMLChannel(replicateCtx, info, entity) - } - return entity, nil } else if strings.ToLower(targetConfig.TargetDBType) == "bigquery" { dataHandler, err := cdcwriter.NewBigQueryDataHandler( cdcwriter.ProjectOption(targetConfig.ProjectId), @@ -643,31 +623,10 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err taskLog.Warn("fail to new the bigquery data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } - writerObj := cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ + writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ MessageBufferSize: bufferSize, Retry: e.config.Retry, }, metaOp.GetAllDroppedObj()) - - e.replicateEntityMap.Lock() - defer e.replicateEntityMap.Unlock() - entity, ok := e.replicateEntityMap.data[milvusAddress] - if !ok { - replicateCtx, cancelReplicateFunc := context.WithCancel(ctx) - channelManager.SetCtx(replicateCtx) - entity = &ReplicateEntity{ - targetClient: milvusClient, - channelManager: channelManager, - metaOp: metaOp, - writerObj: writerObj, - quitFunc: cancelReplicateFunc, - mqDispatcher: msgDispatcherClient, - mqTTDispatcher: msgTTDispatcherClient, - } - e.replicateEntityMap.data[milvusAddress] = entity - e.startReplicateAPIEvent(replicateCtx, info, entity) - e.startReplicateDMLChannel(replicateCtx, info, entity) - } - return entity, nil } else if strings.ToLower(targetConfig.TargetDBType) == "mysql" { dataHandler, err := cdcwriter.NewMySQLDataHandler( cdcwriter.MySQLAddressOption(fmt.Sprintf("%s:%d", targetConfig.Host, targetConfig.Port)), @@ -678,34 +637,32 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err taskLog.Warn("fail to new the mysql data handler", zap.Error(err)) return nil, servererror.NewClientError("fail to new the data handler, task_id: ") } - writerObj := cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ + writerObj = cdcwriter.NewChannelWriter(dataHandler, strings.ToLower(targetConfig.TargetDBType), config.WriterConfig{ MessageBufferSize: bufferSize, Retry: e.config.Retry, }, metaOp.GetAllDroppedObj()) + } - e.replicateEntityMap.Lock() - defer e.replicateEntityMap.Unlock() - entity, ok := e.replicateEntityMap.data[milvusAddress] - if !ok { - replicateCtx, cancelReplicateFunc := context.WithCancel(ctx) - channelManager.SetCtx(replicateCtx) - entity = &ReplicateEntity{ - targetClient: milvusClient, - channelManager: channelManager, - metaOp: metaOp, - writerObj: writerObj, - quitFunc: cancelReplicateFunc, - mqDispatcher: msgDispatcherClient, - mqTTDispatcher: msgTTDispatcherClient, - } - e.replicateEntityMap.data[milvusAddress] = entity - e.startReplicateAPIEvent(replicateCtx, info, entity) - e.startReplicateDMLChannel(replicateCtx, info, entity) + e.replicateEntityMap.Lock() + defer e.replicateEntityMap.Unlock() + entity, ok := e.replicateEntityMap.data[milvusAddress] + if !ok { + replicateCtx, cancelReplicateFunc := context.WithCancel(ctx) + channelManager.SetCtx(replicateCtx) + entity = &ReplicateEntity{ + targetClient: milvusClient, + channelManager: channelManager, + metaOp: metaOp, + writerObj: writerObj, + quitFunc: cancelReplicateFunc, + mqDispatcher: msgDispatcherClient, + mqTTDispatcher: msgTTDispatcherClient, } - return entity, nil + e.replicateEntityMap.data[milvusAddress] = entity + e.startReplicateAPIEvent(replicateCtx, info, entity) + e.startReplicateDMLChannel(replicateCtx, info, entity) } - - return nil, nil + return entity, nil } func (e *MetaCDC) startReplicateAPIEvent(replicateCtx context.Context, info *meta.TaskInfo, entity *ReplicateEntity) { From 979626ace2b63b0876cc15b7d221fb461571f85d Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 19 Jul 2024 16:17:22 +0900 Subject: [PATCH 041/118] multi target database --- core/reader/replicate_channel_manager.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/reader/replicate_channel_manager.go b/core/reader/replicate_channel_manager.go index 241c15e0..3902fc52 100644 --- a/core/reader/replicate_channel_manager.go +++ b/core/reader/replicate_channel_manager.go @@ -556,6 +556,7 @@ func (r *replicateChannelManager) startReadChannel(sourceInfo *model.SourceColle // TODO how to handle the seek position when the pchannel has been replicated channelHandler, ok := r.channelHandlerMap[sourceInfo.PChannelName] + log.Info("channel handler", zap.Any("handler", channelHandler), zap.Bool("ok", ok)) if !ok { var err error channelHandler, err = initReplicateChannelHandler(r.getCtx(), @@ -582,7 +583,8 @@ func (r *replicateChannelManager) startReadChannel(sourceInfo *model.SourceColle for _, handler := range r.channelHandlerMap { handler.recordLock.RLock() - if handler.targetPChannel == targetInfo.PChannel && (targetInfo.PChannel != "mysql" && targetInfo.PChannel != "bigquery") { + // targetInfo.CollectionID == 0 은 target db가 milvus가 아니다. + if handler.targetPChannel == targetInfo.PChannel && targetInfo.CollectionID != 0 { hasReplicateForTargetChannel = true } handler.recordLock.RUnlock() From bdc6d86781e61aa0fde9addd66fb68acc438f96d Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 19 Jul 2024 16:36:36 +0900 Subject: [PATCH 042/118] multi target database --- core/writer/mysql_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 5421c050..7e379721 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -80,6 +80,7 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e func (m *MySQLDataHandler) mysqlOp(ctx context.Context, query string, args ...interface{}) error { retryFunc := func() error { + log.Info("executing mysql operation", zap.String("query", query), zap.Any("args", args)) _, err := m.db.ExecContext(ctx, query, args...) return err } From 3e8c4946f33eeb5f834643875e1b93a94cf7b44d Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 19 Jul 2024 16:52:57 +0900 Subject: [PATCH 043/118] multi target database --- core/writer/mysql_handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 7e379721..745a9a6e 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -94,11 +94,13 @@ func (m *MySQLDataHandler) mysqlOp(ctx context.Context, query string, args ...in func (m *MySQLDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", param.Schema.CollectionName, param.Schema.Fields) + log.Info(query) return m.mysqlOp(ctx, query) } func (m *MySQLDataHandler) DropCollection(ctx context.Context, param *api.DropCollectionParam) error { query := fmt.Sprintf("DROP TABLE IF EXISTS %s", param.CollectionName) + log.Info(query) return m.mysqlOp(ctx, query) } @@ -111,6 +113,7 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e } query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", param.CollectionName, join(columns, ","), join(values, ",")) + log.Info(query) args := make([]interface{}, len(param.Columns)) for i, col := range param.Columns { args[i] = col.FieldData() From 3fae9617a1c149c40de9279533e697436376bd9f Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 19 Jul 2024 17:06:03 +0900 Subject: [PATCH 044/118] multi target database --- core/writer/mysql_handler.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 745a9a6e..1c5f56fa 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -94,13 +94,13 @@ func (m *MySQLDataHandler) mysqlOp(ctx context.Context, query string, args ...in func (m *MySQLDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", param.Schema.CollectionName, param.Schema.Fields) - log.Info(query) + log.Info("CREATE TABLE", zap.String("query", query)) return m.mysqlOp(ctx, query) } func (m *MySQLDataHandler) DropCollection(ctx context.Context, param *api.DropCollectionParam) error { query := fmt.Sprintf("DROP TABLE IF EXISTS %s", param.CollectionName) - log.Info(query) + log.Info("DROP TABLE", zap.String("query", query)) return m.mysqlOp(ctx, query) } @@ -113,7 +113,7 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e } query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", param.CollectionName, join(columns, ","), join(values, ",")) - log.Info(query) + log.Info("INSERT", zap.String("query", query)) args := make([]interface{}, len(param.Columns)) for i, col := range param.Columns { args[i] = col.FieldData() @@ -123,7 +123,7 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e func (m *MySQLDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { query := fmt.Sprintf("DELETE FROM %s WHERE %s = ?", param.CollectionName, param.Column.Name) - log.Info(query) + log.Info("DELETE", zap.String("query", query)) return m.mysqlOp(ctx, query, param.Column.FieldData()) } @@ -140,7 +140,7 @@ func (m *MySQLDataHandler) DropPartition(ctx context.Context, param *api.DropPar func (m *MySQLDataHandler) CreateIndex(ctx context.Context, param *api.CreateIndexParam) error { query := fmt.Sprintf("CREATE INDEX %s ON %s (%s)", param.GetIndexName(), param.GetCollectionName(), param.GetFieldName()) - log.Info(query) + log.Info("CREATE INDEX", zap.String("query", query)) return m.mysqlOp(ctx, query) } @@ -176,13 +176,13 @@ func (m *MySQLDataHandler) Flush(ctx context.Context, param *api.FlushParam) err func (m *MySQLDataHandler) CreateDatabase(ctx context.Context, param *api.CreateDatabaseParam) error { query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", param.DbName) - log.Info(query) + log.Info("CREATE DATABASE", zap.String("query", query)) return m.mysqlOp(ctx, query) } func (m *MySQLDataHandler) DropDatabase(ctx context.Context, param *api.DropDatabaseParam) error { query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", param.DbName) - log.Info(query) + log.Info("DROP DATABASE", zap.String("query", query)) return m.mysqlOp(ctx, query) } @@ -214,7 +214,7 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl func (m *MySQLDataHandler) DescribeCollection(ctx context.Context, param *api.DescribeCollectionParam) error { query := fmt.Sprintf("DESCRIBE %s", param.Name) - log.Info(query) + log.Info("DESCRIBE", zap.String("query", query)) return m.mysqlOp(ctx, query) } From b322c6c0d93fdb84751c899628b59cc3cfee3624 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 22 Jul 2024 17:30:28 +0900 Subject: [PATCH 045/118] multi target database --- core/writer/channel_writer.go | 1 + core/writer/milvus_handler.go | 2 ++ core/writer/mysql_handler.go | 2 +- server/go.sum | 17 +++++++++++++++-- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/core/writer/channel_writer.go b/core/writer/channel_writer.go index 59dbfff1..f9a0b3a1 100644 --- a/core/writer/channel_writer.go +++ b/core/writer/channel_writer.go @@ -151,6 +151,7 @@ func (c *ChannelWriter) HandleReplicateMessage(ctx context.Context, channelName } if msg.Type() == commonpb.MsgType_Insert { insertMsg := msg.(*msgstream.InsertMsg) + logFields = append(logFields, zap.String("collection", insertMsg.GetCollectionName()), zap.String("partition", insertMsg.GetPartitionName()), diff --git a/core/writer/milvus_handler.go b/core/writer/milvus_handler.go index 120bd1c6..1023ccaa 100644 --- a/core/writer/milvus_handler.go +++ b/core/writer/milvus_handler.go @@ -130,6 +130,8 @@ func (m *MilvusDataHandler) Insert(ctx context.Context, param *api.InsertParam) log.Info("ignore partition name in insert request", zap.String("partition", partitionName)) partitionName = "" } + + log.Info("insert request", zap.String("collection", param.CollectionName), zap.String("partition", partitionName), zap.Any("columns", param.Columns)) return m.milvusOp(ctx, param.Database, func(milvus client.Client) error { _, err := milvus.Insert(ctx, param.CollectionName, partitionName, param.Columns...) return err diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 1c5f56fa..713137fb 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -146,7 +146,7 @@ func (m *MySQLDataHandler) CreateIndex(ctx context.Context, param *api.CreateInd func (m *MySQLDataHandler) DropIndex(ctx context.Context, param *api.DropIndexParam) error { query := fmt.Sprintf("DROP INDEX %s ON %s", param.IndexName, param.CollectionName) - log.Info(query) + log.Info("DROP INDEX", zap.String("query", query)) return m.mysqlOp(ctx, query) } diff --git a/server/go.sum b/server/go.sum index ca164f4a..59c033fb 100644 --- a/server/go.sum +++ b/server/go.sum @@ -32,11 +32,15 @@ cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvu cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP4c52gw= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -46,6 +50,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= @@ -62,6 +68,8 @@ github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -336,9 +344,12 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -462,7 +473,6 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -786,6 +796,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -997,7 +1009,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= @@ -1171,6 +1182,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= From 387a4b6d9378cddc8d52c781435f79bcdef5e265 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 22 Jul 2024 18:33:41 +0900 Subject: [PATCH 046/118] multi target database --- core/writer/milvus_handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/writer/milvus_handler.go b/core/writer/milvus_handler.go index 1023ccaa..20f91ad3 100644 --- a/core/writer/milvus_handler.go +++ b/core/writer/milvus_handler.go @@ -267,6 +267,8 @@ func (m *MilvusDataHandler) ReplicateMessage(ctx context.Context, param *api.Rep err error opErr error ) + + log.Info("replicate message", zap.String("channel", param.ChannelName), zap.Any("base", param.Base), zap.Any("msgs", param.MsgsBytes), zap.Any("WithReplicateMessageMsgBase", client.WithReplicateMessageMsgBase(param.Base))) opErr = m.milvusOp(ctx, "", func(milvus client.Client) error { resp, err = milvus.ReplicateMessage(ctx, param.ChannelName, param.BeginTs, param.EndTs, From 1a27370649fd46aae26cf1918202a1aa1214c275 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 22 Jul 2024 18:46:11 +0900 Subject: [PATCH 047/118] multi target database --- core/writer/milvus_handler.go | 38 ++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/core/writer/milvus_handler.go b/core/writer/milvus_handler.go index 20f91ad3..24c0e963 100644 --- a/core/writer/milvus_handler.go +++ b/core/writer/milvus_handler.go @@ -263,26 +263,28 @@ func (m *MilvusDataHandler) DropDatabase(ctx context.Context, param *api.DropDat func (m *MilvusDataHandler) ReplicateMessage(ctx context.Context, param *api.ReplicateMessageParam) error { var ( - resp *entity.MessageInfo - err error - opErr error + resp *entity.MessageInfo + // err error + // opErr error ) - log.Info("replicate message", zap.String("channel", param.ChannelName), zap.Any("base", param.Base), zap.Any("msgs", param.MsgsBytes), zap.Any("WithReplicateMessageMsgBase", client.WithReplicateMessageMsgBase(param.Base))) - opErr = m.milvusOp(ctx, "", func(milvus client.Client) error { - resp, err = milvus.ReplicateMessage(ctx, param.ChannelName, - param.BeginTs, param.EndTs, - param.MsgsBytes, - param.StartPositions, param.EndPositions, - client.WithReplicateMessageMsgBase(param.Base)) - return err - }) - if err != nil { - return err - } - if opErr != nil { - return opErr - } + /* + opErr = m.milvusOp(ctx, "", func(milvus client.Client) error { + resp, err = milvus.ReplicateMessage(ctx, param.ChannelName, + param.BeginTs, param.EndTs, + param.MsgsBytes, + param.StartPositions, param.EndPositions, + client.WithReplicateMessageMsgBase(param.Base)) + return err + }) + if err != nil { + return err + } + if opErr != nil { + return opErr + } + + */ param.TargetMsgPosition = resp.Position return nil } From 7b379fa2e247230ca5cd69609dc30681812cf045 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 22 Jul 2024 18:54:59 +0900 Subject: [PATCH 048/118] multi target database --- core/writer/milvus_handler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/writer/milvus_handler.go b/core/writer/milvus_handler.go index 24c0e963..05bad98a 100644 --- a/core/writer/milvus_handler.go +++ b/core/writer/milvus_handler.go @@ -263,9 +263,9 @@ func (m *MilvusDataHandler) DropDatabase(ctx context.Context, param *api.DropDat func (m *MilvusDataHandler) ReplicateMessage(ctx context.Context, param *api.ReplicateMessageParam) error { var ( - resp *entity.MessageInfo - // err error - // opErr error + // resp *entity.MessageInfo + // err error + // opErr error ) /* @@ -285,7 +285,7 @@ func (m *MilvusDataHandler) ReplicateMessage(ctx context.Context, param *api.Rep } */ - param.TargetMsgPosition = resp.Position + //param.TargetMsgPosition = resp.Position return nil } From 2da523d5729047729863540129a41818fbb1a809 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 22 Jul 2024 20:02:44 +0900 Subject: [PATCH 049/118] multi target database --- core/writer/milvus_handler.go | 45 ++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/core/writer/milvus_handler.go b/core/writer/milvus_handler.go index 05bad98a..f9f1a8bb 100644 --- a/core/writer/milvus_handler.go +++ b/core/writer/milvus_handler.go @@ -20,6 +20,7 @@ package writer import ( "context" + "fmt" "time" "github.com/cockroachdb/errors" @@ -29,6 +30,7 @@ import ( "github.com/milvus-io/milvus-sdk-go/v2/entity" "github.com/milvus-io/milvus/pkg/util/retry" + "encoding/json" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" @@ -263,29 +265,34 @@ func (m *MilvusDataHandler) DropDatabase(ctx context.Context, param *api.DropDat func (m *MilvusDataHandler) ReplicateMessage(ctx context.Context, param *api.ReplicateMessageParam) error { var ( - // resp *entity.MessageInfo - // err error - // opErr error + resp *entity.MessageInfo + err error + opErr error ) - /* - opErr = m.milvusOp(ctx, "", func(milvus client.Client) error { - resp, err = milvus.ReplicateMessage(ctx, param.ChannelName, - param.BeginTs, param.EndTs, - param.MsgsBytes, - param.StartPositions, param.EndPositions, - client.WithReplicateMessageMsgBase(param.Base)) - return err - }) - if err != nil { - return err - } - if opErr != nil { - return opErr + for _, msgBytes := range param.MsgsBytes { + var record map[string]interface{} + if err := json.Unmarshal(msgBytes, &record); err != nil { + return fmt.Errorf("failed to decode msgsBytes: %v", err) } + log.Info("msgBytes", zap.Any("msgBytes", record)) + } - */ - //param.TargetMsgPosition = resp.Position + opErr = m.milvusOp(ctx, "", func(milvus client.Client) error { + resp, err = milvus.ReplicateMessage(ctx, param.ChannelName, + param.BeginTs, param.EndTs, + param.MsgsBytes, + param.StartPositions, param.EndPositions, + client.WithReplicateMessageMsgBase(param.Base)) + return err + }) + if err != nil { + return err + } + if opErr != nil { + return opErr + } + param.TargetMsgPosition = resp.Position return nil } From b7cf494fff7b6554a31be185d9a77c6a9c664274 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 24 Jul 2024 18:53:25 +0900 Subject: [PATCH 050/118] multi target database --- core/writer/milvus_handler.go | 186 ++++++++++++++++++++++++++++++++-- core/writer/mysql_handler.go | 110 ++++++++++++++++---- 2 files changed, 266 insertions(+), 30 deletions(-) diff --git a/core/writer/milvus_handler.go b/core/writer/milvus_handler.go index f9f1a8bb..83a18cec 100644 --- a/core/writer/milvus_handler.go +++ b/core/writer/milvus_handler.go @@ -19,8 +19,13 @@ package writer import ( + "bytes" "context" + "encoding/binary" "fmt" + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/pkg/mq/msgstream" "time" "github.com/cockroachdb/errors" @@ -30,7 +35,6 @@ import ( "github.com/milvus-io/milvus-sdk-go/v2/entity" "github.com/milvus-io/milvus/pkg/util/retry" - "encoding/json" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" @@ -270,14 +274,6 @@ func (m *MilvusDataHandler) ReplicateMessage(ctx context.Context, param *api.Rep opErr error ) - for _, msgBytes := range param.MsgsBytes { - var record map[string]interface{} - if err := json.Unmarshal(msgBytes, &record); err != nil { - return fmt.Errorf("failed to decode msgsBytes: %v", err) - } - log.Info("msgBytes", zap.Any("msgBytes", record)) - } - opErr = m.milvusOp(ctx, "", func(milvus client.Client) error { resp, err = milvus.ReplicateMessage(ctx, param.ChannelName, param.BeginTs, param.EndTs, @@ -296,6 +292,178 @@ func (m *MilvusDataHandler) ReplicateMessage(ctx context.Context, param *api.Rep return nil } +func convertBlobData(blob *commonpb.Blob, convertType string) (interface{}, error) { + if blob == nil || len(blob.Value) == 0 { + return nil, fmt.Errorf("empty or nil blob") + } + + buf := bytes.NewReader(blob.Value) + + if convertType == "floatvector" { + // int64 변환 + var intValue int64 + if err := binary.Read(buf, binary.LittleEndian, &intValue); err != nil { + return nil, fmt.Errorf("failed to read int64: %v", err) + } + + return intValue, nil + } + + // bool 변환 + var boolValue bool + if convertType == "bool" { + if err := binary.Read(buf, binary.LittleEndian, &boolValue); err != nil { + return nil, fmt.Errorf("failed to read bool: %v", err) + } + } + + // varchar 변환 + if convertType == "varchar" { + stringBytes := make([]byte, 0) + var stringValue string + + for { + b, err := buf.ReadByte() + if err != nil { + return nil, fmt.Errorf("failed to read string: %v", err) + } + if b == 0 { + break + } + stringBytes = append(stringBytes, b) + } + stringValue = string(stringBytes) + + return stringValue, nil + } + + // [][]float32 변환 + if convertType == "floatvector" { + var floatArrays [][]float32 + + for buf.Len() > 0 { + var floatArray []float32 + for i := 0; i < 3; i++ { // 예제에서는 각 float32 배열이 3개의 요소를 가진다고 가정 + var floatValue float32 + if err := binary.Read(buf, binary.LittleEndian, &floatValue); err != nil { + return nil, fmt.Errorf("failed to read float32: %v", err) + } + floatArray = append(floatArray, floatValue) + } + floatArrays = append(floatArrays, floatArray) + } + + return floatArrays, nil + } + + return nil, nil +} + +func convertUintSliceToIntSlice(uintSlice []uint64) []int64 { + intSlice := make([]int64, len(uintSlice)) + for i, v := range uintSlice { + if v > uint64(^int64(0)) { + return nil + } + intSlice[i] = int64(v) + } + return intSlice +} + +// convertInsertMsgToInsertParam는 msgstream.InsertMsg를 api.InsertParam으로 변환합니다. +func convertInsertMsgToInsertParam(insertMsg *msgstream.InsertMsg) (*api.InsertParam, error) { + if insertMsg == nil { + return nil, fmt.Errorf("nil insert message") + } + + collectionName := insertMsg.CollectionName + + // 각 필드 데이터를 추출하여 entity.Column 타입으로 변환합니다. + var columns []entity.Column + + // 예제에서는 RowIDs와 Timestamps를 사용합니다. + rowIDColumn := entity.NewColumnInt64("row_id", insertMsg.RowIDs) + timestampColumn := entity.NewColumnInt64("timestamp", convertUintSliceToIntSlice(insertMsg.Timestamps)) + columns = append(columns, rowIDColumn, timestampColumn) + + // InsertMsg의 FieldsData를 entity.Column으로 변환합니다. + for _, fieldData := range insertMsg.FieldsData { + switch fieldData.Type { + case schemapb.DataType_Bool: + values := make([]bool, len(fieldData.GetScalars().GetBoolData().Data)) + for i, v := range fieldData.GetScalars().GetBoolData().Data { + values[i] = v + } + column := entity.NewColumnBool(fieldData.FieldName, values) + columns = append(columns, column) + case schemapb.DataType_VarChar: + values := make([]string, len(fieldData.GetScalars().GetStringData().Data)) + for i, v := range fieldData.GetScalars().GetStringData().Data { + values[i] = v + } + column := entity.NewColumnString(fieldData.FieldName, values) + columns = append(columns, column) + case schemapb.DataType_Int64: + values := make([]int64, len(fieldData.GetScalars().GetLongData().Data)) + for i, v := range fieldData.GetScalars().GetLongData().Data { + values[i] = v + } + column := entity.NewColumnInt64(fieldData.FieldName, values) + columns = append(columns, column) + case schemapb.DataType_Float: + values := make([]float32, len(fieldData.GetScalars().GetFloatData().Data)) + for i, v := range fieldData.GetScalars().GetFloatData().Data { + values[i] = v + } + column := entity.NewColumnFloat(fieldData.FieldName, values) + columns = append(columns, column) + case schemapb.DataType_FloatVector: + dim := int(fieldData.GetVectors().Dim) + values := make([][]float32, len(fieldData.GetVectors().GetFloatVector().Data)/dim) + for i := 0; i < len(values); i++ { + values[i] = fieldData.GetVectors().GetFloatVector().Data[i*dim : (i+1)*dim] + } + column := entity.NewColumnFloatVector(fieldData.FieldName, dim, values) + columns = append(columns, column) + default: + return nil, fmt.Errorf("unsupported field type: %v", fieldData.Type) + } + } + + insertParam := &api.InsertParam{ + CollectionName: collectionName, + Columns: columns, + } + return insertParam, nil +} + +// convertInsertMsgToInsertParam는 msgstream.InsertMsg를 api.InsertParam으로 변환합니다. +func convertDeleteMsgToDeleteParam(deleteMsg *msgstream.DeleteMsg) (*api.DeleteParam, error) { + if deleteMsg == nil { + return nil, fmt.Errorf("nil insert message") + } + + collectionName := deleteMsg.CollectionName + + // PrimaryKeys 필드를 entity.Column으로 변환합니다. + var pkColumn entity.Column + if deleteMsg.PrimaryKeys.GetIntId() != nil { + pkColumn = entity.NewColumnInt64("row_id", deleteMsg.PrimaryKeys.GetIntId().Data) + } else if deleteMsg.PrimaryKeys.GetStrId() != nil { + pkColumn = entity.NewColumnString("row_id", deleteMsg.PrimaryKeys.GetStrId().Data) + } else { + return nil, fmt.Errorf("unsupported primary key data type") + } + + deleteParam := &api.DeleteParam{ + CollectionName: collectionName, + PartitionName: deleteMsg.PartitionName, + Column: pkColumn, + } + + return deleteParam, nil +} + func (m *MilvusDataHandler) DescribeCollection(ctx context.Context, param *api.DescribeCollectionParam) error { return m.milvusOp(ctx, param.Database, func(milvus client.Client) error { _, err := milvus.DescribeCollection(ctx, param.Name) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 713137fb..298a2cc2 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -6,14 +6,15 @@ import ( "fmt" "github.com/cenkalti/backoff/v4" "github.com/go-sql-driver/mysql" - "github.com/milvus-io/milvus-sdk-go/v2/client" - "github.com/milvus-io/milvus-sdk-go/v2/entity" + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus/pkg/mq/msgstream" "strings" "time" "github.com/cockroachdb/errors" "go.uber.org/zap" + "github.com/golang/protobuf/proto" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" @@ -186,29 +187,96 @@ func (m *MySQLDataHandler) DropDatabase(ctx context.Context, param *api.DropData return m.mysqlOp(ctx, query) } -func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.ReplicateMessageParam) error { - var ( - resp *entity.MessageInfo - err error - opErr error - ) - log.Info("ReplicateMessage", zap.Any("param", param)) +func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb.MsgType, msgBytes []byte) error { + //var tsMsg msgstream.TsMsg + var err error - opErr = m.mysqlOp(ctx, "", func(mysqlCli client.Client) error { - resp, err = mysqlCli.ReplicateMessage(ctx, param.ChannelName, - param.BeginTs, param.EndTs, - param.MsgsBytes, - param.StartPositions, param.EndPositions, - client.WithReplicateMessageMsgBase(param.Base)) - return err - }) - if err != nil { + switch msgType { + case commonpb.MsgType_Insert: + insertMsg := &msgstream.InsertMsg{} + err = proto.Unmarshal(msgBytes, insertMsg) + + msg, err := convertInsertMsgToInsertParam(insertMsg) + err = m.Insert(ctx, msg) + if err != nil { + return err + } + case commonpb.MsgType_Delete: + deleteMsg := &msgstream.DeleteMsg{} + err = proto.Unmarshal(msgBytes, deleteMsg) + + msg, err := convertDeleteMsgToDeleteParam(deleteMsg) + if err != nil { + return err + } + err = m.Delete(ctx, msg) + if err != nil { + return err + } + case commonpb.MsgType_Upsert: + // UpsertMsg는 InsertMsg와 DeleteMsg를 포함하므로, 따로 언마샬링한 후 조립합니다. + insertMsg := &msgstream.InsertMsg{} + deleteMsg := &msgstream.DeleteMsg{} + + // msgBytes를 분할하여 각 메시지에 언마샬링합니다. (여기서는 단순히 msgBytes를 나눠서 가정합니다.) + half := len(msgBytes) / 2 + err = proto.Unmarshal(msgBytes[:half], insertMsg) + if err != nil { + return err + } + + imsg, err := convertInsertMsgToInsertParam(insertMsg) + err = m.Insert(ctx, imsg) + if err != nil { + return err + } + + err = proto.Unmarshal(msgBytes[half:], deleteMsg) + if err != nil { + return err + } + + dmsg, err := convertDeleteMsgToDeleteParam(deleteMsg) + if err != nil { + return err + } + err = m.Delete(ctx, dmsg) + if err != nil { + return err + } + default: + err = fmt.Errorf("unsupported message type: %v", msgType) return err } - if opErr != nil { - return opErr + + return nil +} + +func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.ReplicateMessageParam) error { + + log.Info("ReplicateMessage", zap.Any("param", param)) + + for i, msgBytes := range param.MsgsBytes { + header := &commonpb.MsgHeader{} + err := proto.Unmarshal(msgBytes, header) + if err != nil { + log.Warn("failed to unmarshal msg header", zap.Int("index", i), zap.Error(err)) + return err + } + + if header.GetBase() == nil { + log.Warn("msg header base is nil", zap.Int("index", i)) + return err + } + + err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), msgBytes) + if err != nil { + log.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err)) + return err + } + } - param.TargetMsgPosition = resp.Position + return nil } From 89fdd90d6bc9a03ed5c8129611a7d83b60ed8cdf Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 24 Jul 2024 19:04:39 +0900 Subject: [PATCH 051/118] multi target database --- core/writer/milvus_handler.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/writer/milvus_handler.go b/core/writer/milvus_handler.go index 83a18cec..27a4fc38 100644 --- a/core/writer/milvus_handler.go +++ b/core/writer/milvus_handler.go @@ -359,6 +359,7 @@ func convertBlobData(blob *commonpb.Blob, convertType string) (interface{}, erro return nil, nil } +/* func convertUintSliceToIntSlice(uintSlice []uint64) []int64 { intSlice := make([]int64, len(uintSlice)) for i, v := range uintSlice { @@ -369,7 +370,7 @@ func convertUintSliceToIntSlice(uintSlice []uint64) []int64 { } return intSlice } - +*/ // convertInsertMsgToInsertParam는 msgstream.InsertMsg를 api.InsertParam으로 변환합니다. func convertInsertMsgToInsertParam(insertMsg *msgstream.InsertMsg) (*api.InsertParam, error) { if insertMsg == nil { @@ -382,9 +383,9 @@ func convertInsertMsgToInsertParam(insertMsg *msgstream.InsertMsg) (*api.InsertP var columns []entity.Column // 예제에서는 RowIDs와 Timestamps를 사용합니다. - rowIDColumn := entity.NewColumnInt64("row_id", insertMsg.RowIDs) - timestampColumn := entity.NewColumnInt64("timestamp", convertUintSliceToIntSlice(insertMsg.Timestamps)) - columns = append(columns, rowIDColumn, timestampColumn) + // rowIDColumn := entity.NewColumnInt64("row_id", insertMsg.RowIDs) + // timestampColumn := entity.NewColumnInt64("timestamp", convertUintSliceToIntSlice(insertMsg.Timestamps)) + // columns = append(columns, rowIDColumn, timestampColumn) // InsertMsg의 FieldsData를 entity.Column으로 변환합니다. for _, fieldData := range insertMsg.FieldsData { From fb83533967d97f269fede460aff60f02bbdf3e0f Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 24 Jul 2024 19:30:42 +0900 Subject: [PATCH 052/118] multi target database --- core/writer/mysql_handler.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 298a2cc2..d7ce612d 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -195,10 +195,20 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. case commonpb.MsgType_Insert: insertMsg := &msgstream.InsertMsg{} err = proto.Unmarshal(msgBytes, insertMsg) + if err != nil { + log.Warn("failed to unmarshal insert msg", zap.Error(err)) + return err + } msg, err := convertInsertMsgToInsertParam(insertMsg) + if err != nil { + log.Warn("failed to convert insert msg to insert param", zap.Error(err)) + return err + } + err = m.Insert(ctx, msg) if err != nil { + log.Warn("failed to unmarshal insert msg", zap.Error(err)) return err } case commonpb.MsgType_Delete: @@ -207,10 +217,12 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. msg, err := convertDeleteMsgToDeleteParam(deleteMsg) if err != nil { + log.Warn("failed to convert delete msg to delete param", zap.Error(err)) return err } err = m.Delete(ctx, msg) if err != nil { + log.Warn("failed to delete", zap.Error(err)) return err } case commonpb.MsgType_Upsert: @@ -222,12 +234,19 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. half := len(msgBytes) / 2 err = proto.Unmarshal(msgBytes[:half], insertMsg) if err != nil { + log.Warn("failed to unmarshal insert msg", zap.Error(err)) return err } imsg, err := convertInsertMsgToInsertParam(insertMsg) + if err != nil { + log.Warn("failed to convert insert msg to insert param", zap.Error(err)) + return err + } + err = m.Insert(ctx, imsg) if err != nil { + log.Warn("failed to insert", zap.Error(err)) return err } @@ -238,13 +257,17 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. dmsg, err := convertDeleteMsgToDeleteParam(deleteMsg) if err != nil { + log.Warn("failed to convert delete msg to delete param", zap.Error(err)) return err } + err = m.Delete(ctx, dmsg) if err != nil { + log.Warn("failed to delete", zap.Error(err)) return err } default: + log.Warn("unsupported message type", zap.Any("msgType", msgType)) err = fmt.Errorf("unsupported message type: %v", msgType) return err } From 9c206d970e20542caf55bc084415cd99e048f41f Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 24 Jul 2024 19:47:05 +0900 Subject: [PATCH 053/118] multi target database --- core/writer/mysql_handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index d7ce612d..44613499 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -266,6 +266,8 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. log.Warn("failed to delete", zap.Error(err)) return err } + case commonpb.MsgType_TimeTick: + return nil default: log.Warn("unsupported message type", zap.Any("msgType", msgType)) err = fmt.Errorf("unsupported message type: %v", msgType) From aec88762381f867c8e467611a507476db2cb577b Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 25 Jul 2024 17:25:16 +0900 Subject: [PATCH 054/118] multi target database --- core/reader/replicate_channel_manager.go | 14 +++++++++----- core/writer/mysql_handler.go | 3 --- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/core/reader/replicate_channel_manager.go b/core/reader/replicate_channel_manager.go index 3902fc52..be9a231b 100644 --- a/core/reader/replicate_channel_manager.go +++ b/core/reader/replicate_channel_manager.go @@ -1228,11 +1228,15 @@ func (r *replicateChannelHandler) handlePack(forward bool, pack *msgstream.MsgPa } realMsg.CollectionID = info.CollectionID partitionID := realMsg.PartitionID - realMsg.PartitionID, err = r.getPartitionID(sourceCollectionID, partitionID, info, realMsg.PartitionName) - if realMsg.PartitionID == -1 { - log.Info("skip insert msg because partition has been dropped in the source and target", - zap.Int64("partition_id", partitionID), zap.String("partition_name", realMsg.PartitionName)) - continue + + log.Info("target Info", zap.Any("info", info)) + if info.CollectionID != 0 { + realMsg.PartitionID, err = r.getPartitionID(sourceCollectionID, partitionID, info, realMsg.PartitionName) + if realMsg.PartitionID == -1 { + log.Info("skip insert msg because partition has been dropped in the source and target", + zap.Int64("partition_id", partitionID), zap.String("partition_name", realMsg.PartitionName)) + continue + } } realMsg.ShardName = info.VChannel dataLen = int(realMsg.GetNumRows()) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 44613499..a86cc7a3 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -278,9 +278,6 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. } func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.ReplicateMessageParam) error { - - log.Info("ReplicateMessage", zap.Any("param", param)) - for i, msgBytes := range param.MsgsBytes { header := &commonpb.MsgHeader{} err := proto.Unmarshal(msgBytes, header) From e92bf92a39c6980ef5033cddfd6d3742f8266a9e Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 25 Jul 2024 17:59:19 +0900 Subject: [PATCH 055/118] multi target database --- core/writer/mysql_handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index a86cc7a3..a4eed721 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -291,6 +291,8 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl return err } + log.Info("param", zap.Any("param", param)) + log.Info("replicate message", zap.Any("msgType", header.GetBase().GetMsgType()), zap.Any("header", header)) err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), msgBytes) if err != nil { log.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err)) From d9e4dccee7734c28938b78bfdd2db4a3e12720f0 Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 25 Jul 2024 18:22:51 +0900 Subject: [PATCH 056/118] multi target database --- core/writer/mysql_handler.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index a4eed721..a55421a2 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -200,12 +200,14 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. return err } + log.Info("insert msg", zap.Any("insertMsg", insertMsg)) msg, err := convertInsertMsgToInsertParam(insertMsg) if err != nil { log.Warn("failed to convert insert msg to insert param", zap.Error(err)) return err } + log.Info("insert msg param", zap.Any("insertMsg", msg)) err = m.Insert(ctx, msg) if err != nil { log.Warn("failed to unmarshal insert msg", zap.Error(err)) From e4e1fc0825b11a26cde0c4f479788edab2f26b86 Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 25 Jul 2024 19:01:01 +0900 Subject: [PATCH 057/118] multi target database --- core/writer/mysql_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index a55421a2..5cbd3f02 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -200,7 +200,7 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. return err } - log.Info("insert msg", zap.Any("insertMsg", insertMsg)) + log.Info("insert msg", zap.Any("insertMsg", insertMsg.FieldsData), zap.Any("insertMsgGet", insertMsg.GetFieldsData())) msg, err := convertInsertMsgToInsertParam(insertMsg) if err != nil { log.Warn("failed to convert insert msg to insert param", zap.Error(err)) From 983eb4c68436d766c25e7e0585ed371bf53173fd Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 25 Jul 2024 19:28:03 +0900 Subject: [PATCH 058/118] multi target database --- core/writer/mysql_handler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 5cbd3f02..bd904cf4 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -200,7 +200,8 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. return err } - log.Info("insert msg", zap.Any("insertMsg", insertMsg.FieldsData), zap.Any("insertMsgGet", insertMsg.GetFieldsData())) + log.Info("insert msg", zap.Any("insertMsg", insertMsg.InsertRequest), zap.Any("insertMsgRows", insertMsg.NumRows)) + log.Info("msgBytes", zap.Any("msgBytes", msgBytes)) msg, err := convertInsertMsgToInsertParam(insertMsg) if err != nil { log.Warn("failed to convert insert msg to insert param", zap.Error(err)) From 409e8c3872847edef18d49c84cd0442b4a1ce3e2 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 26 Jul 2024 11:26:27 +0900 Subject: [PATCH 059/118] multi target database --- core/writer/channel_writer.go | 2 ++ core/writer/mysql_handler.go | 29 +++++++++++++++-------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/core/writer/channel_writer.go b/core/writer/channel_writer.go index f9a0b3a1..397850d3 100644 --- a/core/writer/channel_writer.go +++ b/core/writer/channel_writer.go @@ -149,6 +149,7 @@ func (c *ChannelWriter) HandleReplicateMessage(ctx context.Context, channelName zap.String("channel", channelName), zap.String("type", msg.Type().String()), } + if msg.Type() == commonpb.MsgType_Insert { insertMsg := msg.(*msgstream.InsertMsg) @@ -175,6 +176,7 @@ func (c *ChannelWriter) HandleReplicateMessage(ctx context.Context, channelName log.Warn("failed to marshal msg", zap.Error(err)) return nil, nil, err } + if _, ok := msgBytes.([]byte); !ok { log.Warn("failed to convert msg bytes to []byte") return nil, nil, err diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index bd904cf4..dc35b60b 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -188,42 +188,43 @@ func (m *MySQLDataHandler) DropDatabase(ctx context.Context, param *api.DropData } func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb.MsgType, msgBytes []byte) error { - //var tsMsg msgstream.TsMsg + var tsMsg msgstream.TsMsg var err error + msg, err := tsMsg.Unmarshal(msgBytes) + if err != nil { + log.Warn("failed to unmarshal ts msg", zap.Error(err)) + return err + } + + log.Info("unmarshalTsMsg", zap.Any("msgType", msgType), zap.Any("msg", msg)) switch msgType { case commonpb.MsgType_Insert: - insertMsg := &msgstream.InsertMsg{} - err = proto.Unmarshal(msgBytes, insertMsg) - if err != nil { - log.Warn("failed to unmarshal insert msg", zap.Error(err)) - return err - } + insertMsg := msg.(*msgstream.InsertMsg) log.Info("insert msg", zap.Any("insertMsg", insertMsg.InsertRequest), zap.Any("insertMsgRows", insertMsg.NumRows)) log.Info("msgBytes", zap.Any("msgBytes", msgBytes)) - msg, err := convertInsertMsgToInsertParam(insertMsg) + tmsg, err := convertInsertMsgToInsertParam(insertMsg) if err != nil { log.Warn("failed to convert insert msg to insert param", zap.Error(err)) return err } - log.Info("insert msg param", zap.Any("insertMsg", msg)) - err = m.Insert(ctx, msg) + log.Info("insert msg param", zap.Any("insertMsg", tmsg)) + err = m.Insert(ctx, tmsg) if err != nil { log.Warn("failed to unmarshal insert msg", zap.Error(err)) return err } case commonpb.MsgType_Delete: - deleteMsg := &msgstream.DeleteMsg{} - err = proto.Unmarshal(msgBytes, deleteMsg) + deleteMsg := msg.(*msgstream.DeleteMsg) - msg, err := convertDeleteMsgToDeleteParam(deleteMsg) + tmsg, err := convertDeleteMsgToDeleteParam(deleteMsg) if err != nil { log.Warn("failed to convert delete msg to delete param", zap.Error(err)) return err } - err = m.Delete(ctx, msg) + err = m.Delete(ctx, tmsg) if err != nil { log.Warn("failed to delete", zap.Error(err)) return err From 73fe2445551eae8e165d2e0a6bbf63e4b0b06519 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 26 Jul 2024 11:39:08 +0900 Subject: [PATCH 060/118] multi target database --- core/writer/mysql_handler.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index dc35b60b..9ab14fd8 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -191,6 +191,10 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. var tsMsg msgstream.TsMsg var err error + if msgBytes == nil { + return errors.New("msgBytes is nil") + + } msg, err := tsMsg.Unmarshal(msgBytes) if err != nil { log.Warn("failed to unmarshal ts msg", zap.Error(err)) From 8cf8def5bf19d074a6458fffb6865fc68d989d4d Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 26 Jul 2024 14:49:27 +0900 Subject: [PATCH 061/118] multi target database --- core/writer/mysql_handler.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 9ab14fd8..5df6737b 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -192,18 +192,23 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. var err error if msgBytes == nil { + log.Warn("msgBytes is nil") return errors.New("msgBytes is nil") } - msg, err := tsMsg.Unmarshal(msgBytes) - if err != nil { - log.Warn("failed to unmarshal ts msg", zap.Error(err)) - return err - } - log.Info("unmarshalTsMsg", zap.Any("msgType", msgType), zap.Any("msg", msg)) switch msgType { case commonpb.MsgType_Insert: + tsMsg = &msgstream.InsertMsg{} + + msg, err := tsMsg.Unmarshal(msgBytes) + if err != nil { + log.Warn("failed to unmarshal ts msg", zap.Error(err)) + return err + } + + log.Info("unmarshalTsMsg", zap.Any("msgType", msgType), zap.Any("msg", msg)) + insertMsg := msg.(*msgstream.InsertMsg) log.Info("insert msg", zap.Any("insertMsg", insertMsg.InsertRequest), zap.Any("insertMsgRows", insertMsg.NumRows)) @@ -221,6 +226,16 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. return err } case commonpb.MsgType_Delete: + tsMsg = &msgstream.DeleteMsg{} + + msg, err := tsMsg.Unmarshal(msgBytes) + if err != nil { + log.Warn("failed to unmarshal ts msg", zap.Error(err)) + return err + } + + log.Info("unmarshalTsMsg", zap.Any("msgType", msgType), zap.Any("msg", msg)) + deleteMsg := msg.(*msgstream.DeleteMsg) tmsg, err := convertDeleteMsgToDeleteParam(deleteMsg) @@ -235,6 +250,7 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. } case commonpb.MsgType_Upsert: // UpsertMsg는 InsertMsg와 DeleteMsg를 포함하므로, 따로 언마샬링한 후 조립합니다. + // tsMsg = &msgstream.UpsertMsg{} insertMsg := &msgstream.InsertMsg{} deleteMsg := &msgstream.DeleteMsg{} From 3c699f53cf44b14d5c0119c54174a5662dd41b90 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 26 Jul 2024 17:46:42 +0900 Subject: [PATCH 062/118] multi target database --- core/writer/mysql_handler.go | 86 +++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 6 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 5df6737b..725cd44d 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -7,6 +7,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/go-sql-driver/mysql" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" "strings" "time" @@ -108,18 +109,91 @@ func (m *MySQLDataHandler) DropCollection(ctx context.Context, param *api.DropCo func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { columns := []string{} values := []string{} + + rowValues := [][]interface{}{} for _, col := range param.Columns { columns = append(columns, col.Name()) - values = append(values, "?") + + colValues := []interface{}{} + switch data := col.FieldData().GetField().(type) { + case *schemapb.FieldData_Scalars: + switch scalarData := data.Scalars.Data.(type) { + case *schemapb.ScalarField_LongData: + for _, v := range scalarData.LongData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_BoolData: + for _, v := range scalarData.BoolData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_StringData: + for _, v := range scalarData.StringData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + case *schemapb.ScalarField_ArrayData: + for _, v := range scalarData.ArrayData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + case *schemapb.ScalarField_IntData: + for _, v := range scalarData.IntData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_FloatData: + for _, v := range scalarData.FloatData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_DoubleData: + for _, v := range scalarData.DoubleData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_JsonData: + for _, v := range scalarData.JsonData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + case *schemapb.ScalarField_BytesData: + for _, v := range scalarData.BytesData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + default: + return fmt.Errorf("unsupported scalar data type: %T", scalarData) + } + + rowValues = append(rowValues, colValues) + case *schemapb.FieldData_Vectors: + switch vectorData := data.Vectors.Data.(type) { + case *schemapb.VectorField_FloatVector: + for _, v := range vectorData.FloatVector.Data { + colValues = append(colValues, fmt.Sprintf("string_to_vector('%v')", v)) + } + + rowValues = append(rowValues, colValues) + default: + return fmt.Errorf("unsupported vector data type: %T", vectorData) + } + default: + return fmt.Errorf("unsupported field data type: %T", data) + } } - query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", + for _, rowValue := range rowValues { + values = append(values, fmt.Sprintf("(%v)", join(interfaceSliceToStringSlice(rowValue), ","))) + } + + //string_to_vector('[1,2,3]') + query := fmt.Sprintf("INSERT INTO %s (%s) VALUES %s", param.CollectionName, join(columns, ","), join(values, ",")) log.Info("INSERT", zap.String("query", query)) - args := make([]interface{}, len(param.Columns)) - for i, col := range param.Columns { - args[i] = col.FieldData() + + return m.mysqlOp(ctx, query) + +} + +func interfaceSliceToStringSlice(input []interface{}) []string { + var output []string + for _, v := range input { + str, _ := v.(string) + output = append(output, str) } - return m.mysqlOp(ctx, query, args...) + return output } func (m *MySQLDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { From f9646094fab1e0315df3ecb2edb561905939a8f5 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 26 Jul 2024 18:31:23 +0900 Subject: [PATCH 063/118] multi target database --- core/writer/mysql_handler.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 725cd44d..c8c9887d 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -9,6 +9,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" + "strconv" "strings" "time" @@ -162,8 +163,13 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e case *schemapb.FieldData_Vectors: switch vectorData := data.Vectors.Data.(type) { case *schemapb.VectorField_FloatVector: - for _, v := range vectorData.FloatVector.Data { - colValues = append(colValues, fmt.Sprintf("string_to_vector('%v')", v)) + dim := data.Vectors.Dim + var vec []float32 + for i, v := range vectorData.FloatVector.Data { + if int64(i)%dim == 0 && i != 0 { + vec = append(vec, v) + } + colValues = append(colValues, fmt.Sprintf("string_to_vector('[%v]')", join(float32SliceToStringSlice(vec), ","))) } rowValues = append(rowValues, colValues) @@ -196,6 +202,14 @@ func interfaceSliceToStringSlice(input []interface{}) []string { return output } +func float32SliceToStringSlice(input []float32) []string { + output := make([]string, len(input)) + for i, v := range input { + output[i] = strconv.FormatFloat(float64(v), 'f', -1, 32) + } + return output +} + func (m *MySQLDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { query := fmt.Sprintf("DELETE FROM %s WHERE %s = ?", param.CollectionName, param.Column.Name) log.Info("DELETE", zap.String("query", query)) From 4c4604813eb690c93f53430692aa7e055f18a791 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 26 Jul 2024 18:36:55 +0900 Subject: [PATCH 064/118] multi target database --- core/writer/mysql_handler.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index c8c9887d..6fc472aa 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -163,13 +163,15 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e case *schemapb.FieldData_Vectors: switch vectorData := data.Vectors.Data.(type) { case *schemapb.VectorField_FloatVector: - dim := data.Vectors.Dim + dim := data.Vectors.Dim - 1 var vec []float32 for i, v := range vectorData.FloatVector.Data { - if int64(i)%dim == 0 && i != 0 { - vec = append(vec, v) + vec = append(vec, v) + + if int64(i)%dim != 0 && i != 0 { + colValues = append(colValues, fmt.Sprintf("string_to_vector('[%v]')", join(float32SliceToStringSlice(vec), ","))) + vec = []float32{} } - colValues = append(colValues, fmt.Sprintf("string_to_vector('[%v]')", join(float32SliceToStringSlice(vec), ","))) } rowValues = append(rowValues, colValues) From ea88c495384fc2297272ef7292920dea4f19196d Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 26 Jul 2024 19:15:19 +0900 Subject: [PATCH 065/118] multi target database --- core/writer/mysql_handler.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 6fc472aa..42795ccb 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -168,7 +168,7 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e for i, v := range vectorData.FloatVector.Data { vec = append(vec, v) - if int64(i)%dim != 0 && i != 0 { + if int64(i)%dim == 0 && i != 0 { colValues = append(colValues, fmt.Sprintf("string_to_vector('[%v]')", join(float32SliceToStringSlice(vec), ","))) vec = []float32{} } @@ -183,7 +183,14 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e } } for _, rowValue := range rowValues { - values = append(values, fmt.Sprintf("(%v)", join(interfaceSliceToStringSlice(rowValue), ","))) + for _, colValue := range rowValue { + switch colValue.(type) { + case interface{}: + values = append(values, fmt.Sprintf("(%v)", colValue)) + default: + values = append(values, fmt.Sprintf("'%v'", colValue)) + } + } } //string_to_vector('[1,2,3]') From 3a1f5b15726bca026b6ca7ecc53410dc2c65fa44 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 26 Jul 2024 19:41:26 +0900 Subject: [PATCH 066/118] multi target database --- core/writer/mysql_handler.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 42795ccb..05945281 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -182,15 +182,21 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e return fmt.Errorf("unsupported field data type: %T", data) } } + + var value string for _, rowValue := range rowValues { + value = "(" for _, colValue := range rowValue { switch colValue.(type) { - case interface{}: - values = append(values, fmt.Sprintf("(%v)", colValue)) + case string: + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", colValue)) default: + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", colValue)) values = append(values, fmt.Sprintf("'%v'", colValue)) } } + value = fmt.Sprintf("%s)", value) + values = append(values, value) } //string_to_vector('[1,2,3]') From 867d671f59a530ca2910d277fd0ceebe3dbb8cf5 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 26 Jul 2024 19:48:10 +0900 Subject: [PATCH 067/118] multi target database --- core/writer/mysql_handler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 05945281..db9ee47c 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -192,7 +192,6 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", colValue)) default: value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", colValue)) - values = append(values, fmt.Sprintf("'%v'", colValue)) } } value = fmt.Sprintf("%s)", value) From 90628080921a022866498eee836cb0cc72768fd8 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 29 Jul 2024 15:07:10 +0900 Subject: [PATCH 068/118] multi target database --- core/writer/mysql_handler.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index db9ee47c..21462a36 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -109,7 +109,6 @@ func (m *MySQLDataHandler) DropCollection(ctx context.Context, param *api.DropCo func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { columns := []string{} - values := []string{} rowValues := [][]interface{}{} for _, col := range param.Columns { @@ -183,24 +182,30 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e } } - var value string - for _, rowValue := range rowValues { + var value, values string + + for rowCnt := 0; rowCnt < len(rowValues[0]); rowCnt++ { value = "(" - for _, colValue := range rowValue { - switch colValue.(type) { + for colNo, _ := range columns { + switch rowValues[colNo][rowCnt].(type) { case string: - value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", colValue)) + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", rowValues[colNo][rowCnt])) default: - value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", colValue)) + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) } } value = fmt.Sprintf("%s)", value) - values = append(values, value) + + if rowCnt == 0 { + values = fmt.Sprintf("%s", value) + } else { + values = fmt.Sprintf("%s,%s", values, value) + } } //string_to_vector('[1,2,3]') query := fmt.Sprintf("INSERT INTO %s (%s) VALUES %s", - param.CollectionName, join(columns, ","), join(values, ",")) + param.CollectionName, join(columns, ","), values) log.Info("INSERT", zap.String("query", query)) return m.mysqlOp(ctx, query) From 732023dcc8c2f5cb08ee14b55b5c8a9bdd900dbb Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 29 Jul 2024 15:41:48 +0900 Subject: [PATCH 069/118] multi target database --- core/writer/mysql_handler.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 21462a36..b12e14c5 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -85,6 +85,9 @@ func (m *MySQLDataHandler) mysqlOp(ctx context.Context, query string, args ...in retryFunc := func() error { log.Info("executing mysql operation", zap.String("query", query), zap.Any("args", args)) _, err := m.db.ExecContext(ctx, query, args...) + if err != nil { + log.Warn("failed to execute mysql operation", zap.Error(err)) + } return err } @@ -112,7 +115,7 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e rowValues := [][]interface{}{} for _, col := range param.Columns { - columns = append(columns, col.Name()) + columns = append(columns, fmt.Sprintf("`%s`", col.Name())) colValues := []interface{}{} switch data := col.FieldData().GetField().(type) { From 7a42b195cc6877077428041d46c531061316d892 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 29 Jul 2024 16:09:51 +0900 Subject: [PATCH 070/118] multi target database --- core/writer/mysql_handler.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index b12e14c5..cb5bbf0e 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -192,9 +192,17 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e for colNo, _ := range columns { switch rowValues[colNo][rowCnt].(type) { case string: - value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", rowValues[colNo][rowCnt])) + if colNo == 0 { + value = fmt.Sprintf("%s", fmt.Sprintf("%s", rowValues[colNo][rowCnt])) + } else { + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", rowValues[colNo][rowCnt])) + } default: - value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + if colNo == 0 { + value = fmt.Sprintf("%s", fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + } else { + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + } } } value = fmt.Sprintf("%s)", value) @@ -207,8 +215,8 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e } //string_to_vector('[1,2,3]') - query := fmt.Sprintf("INSERT INTO %s (%s) VALUES %s", - param.CollectionName, join(columns, ","), values) + query := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s", + param.Database, param.CollectionName, join(columns, ","), values) log.Info("INSERT", zap.String("query", query)) return m.mysqlOp(ctx, query) @@ -233,7 +241,7 @@ func float32SliceToStringSlice(input []float32) []string { } func (m *MySQLDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { - query := fmt.Sprintf("DELETE FROM %s WHERE %s = ?", param.CollectionName, param.Column.Name) + query := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE `%s` = ?", param.Database, param.CollectionName, param.Column.Name) log.Info("DELETE", zap.String("query", query)) return m.mysqlOp(ctx, query, param.Column.FieldData()) } @@ -249,8 +257,8 @@ func (m *MySQLDataHandler) DropPartition(ctx context.Context, param *api.DropPar } func (m *MySQLDataHandler) CreateIndex(ctx context.Context, param *api.CreateIndexParam) error { - query := fmt.Sprintf("CREATE INDEX %s ON %s (%s)", - param.GetIndexName(), param.GetCollectionName(), param.GetFieldName()) + query := fmt.Sprintf("CREATE INDEX %s ON `%s`.`%s` (`%s`)", + param.GetIndexName(), param.DbName, param.GetCollectionName(), param.GetFieldName()) log.Info("CREATE INDEX", zap.String("query", query)) return m.mysqlOp(ctx, query) } @@ -286,7 +294,7 @@ func (m *MySQLDataHandler) Flush(ctx context.Context, param *api.FlushParam) err } func (m *MySQLDataHandler) CreateDatabase(ctx context.Context, param *api.CreateDatabaseParam) error { - query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", param.DbName) + query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", param.DbName) log.Info("CREATE DATABASE", zap.String("query", query)) return m.mysqlOp(ctx, query) } From 0f65a620bbb3acf12932cee73f098ea9f5c195a1 Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 29 Jul 2024 16:28:29 +0900 Subject: [PATCH 071/118] multi target database --- core/writer/mysql_handler.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index cb5bbf0e..f9a7da5c 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -337,6 +337,7 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. return err } + tmsg.Database = insertMsg.DbName log.Info("insert msg param", zap.Any("insertMsg", tmsg)) err = m.Insert(ctx, tmsg) if err != nil { @@ -361,6 +362,9 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. log.Warn("failed to convert delete msg to delete param", zap.Error(err)) return err } + + tmsg.Database = deleteMsg.DbName + err = m.Delete(ctx, tmsg) if err != nil { log.Warn("failed to delete", zap.Error(err)) @@ -433,8 +437,6 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl return err } - log.Info("param", zap.Any("param", param)) - log.Info("replicate message", zap.Any("msgType", header.GetBase().GetMsgType()), zap.Any("header", header)) err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), msgBytes) if err != nil { log.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err)) From 8c9c9cc205a7b543f4484b631d7a047d84a9327a Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 29 Jul 2024 16:39:46 +0900 Subject: [PATCH 072/118] multi target database --- core/writer/mysql_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index f9a7da5c..5e50af16 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -217,7 +217,7 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e //string_to_vector('[1,2,3]') query := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s", param.Database, param.CollectionName, join(columns, ","), values) - log.Info("INSERT", zap.String("query", query)) + // log.Info("INSERT", zap.String("query", query)) return m.mysqlOp(ctx, query) From 9041bb733b7e775a105fff9560d85379c20d6eea Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 29 Jul 2024 16:55:58 +0900 Subject: [PATCH 073/118] multi target database --- core/writer/mysql_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 5e50af16..17ec5cd7 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -437,6 +437,7 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl return err } + log.Info("replicate message", zap.Any("DbName", param.Database)) err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), msgBytes) if err != nil { log.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err)) From a0230c258805b80738fea7bda0da9537bc34324f Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 29 Jul 2024 20:05:27 +0900 Subject: [PATCH 074/118] multi target database --- core/writer/channel_writer.go | 1 + core/writer/milvus_handler.go | 82 ----------------------------------- core/writer/mysql_handler.go | 10 ++--- server/cdc_impl.go | 2 + 4 files changed, 8 insertions(+), 87 deletions(-) diff --git a/core/writer/channel_writer.go b/core/writer/channel_writer.go index 397850d3..3542586d 100644 --- a/core/writer/channel_writer.go +++ b/core/writer/channel_writer.go @@ -183,6 +183,7 @@ func (c *ChannelWriter) HandleReplicateMessage(ctx context.Context, channelName } msgBytesArr = append(msgBytesArr, msgBytes.([]byte)) } + log.Info("Message Pack", zap.Any("msgPack", msgPack)) replicateMessageParam := &api.ReplicateMessageParam{ MsgBaseParam: api.MsgBaseParam{Base: &commonpb.MsgBase{ReplicateInfo: &commonpb.ReplicateInfo{IsReplicate: true}}}, ChannelName: channelName, diff --git a/core/writer/milvus_handler.go b/core/writer/milvus_handler.go index 27a4fc38..7aac424e 100644 --- a/core/writer/milvus_handler.go +++ b/core/writer/milvus_handler.go @@ -19,11 +19,8 @@ package writer import ( - "bytes" "context" - "encoding/binary" "fmt" - "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" "time" @@ -292,85 +289,6 @@ func (m *MilvusDataHandler) ReplicateMessage(ctx context.Context, param *api.Rep return nil } -func convertBlobData(blob *commonpb.Blob, convertType string) (interface{}, error) { - if blob == nil || len(blob.Value) == 0 { - return nil, fmt.Errorf("empty or nil blob") - } - - buf := bytes.NewReader(blob.Value) - - if convertType == "floatvector" { - // int64 변환 - var intValue int64 - if err := binary.Read(buf, binary.LittleEndian, &intValue); err != nil { - return nil, fmt.Errorf("failed to read int64: %v", err) - } - - return intValue, nil - } - - // bool 변환 - var boolValue bool - if convertType == "bool" { - if err := binary.Read(buf, binary.LittleEndian, &boolValue); err != nil { - return nil, fmt.Errorf("failed to read bool: %v", err) - } - } - - // varchar 변환 - if convertType == "varchar" { - stringBytes := make([]byte, 0) - var stringValue string - - for { - b, err := buf.ReadByte() - if err != nil { - return nil, fmt.Errorf("failed to read string: %v", err) - } - if b == 0 { - break - } - stringBytes = append(stringBytes, b) - } - stringValue = string(stringBytes) - - return stringValue, nil - } - - // [][]float32 변환 - if convertType == "floatvector" { - var floatArrays [][]float32 - - for buf.Len() > 0 { - var floatArray []float32 - for i := 0; i < 3; i++ { // 예제에서는 각 float32 배열이 3개의 요소를 가진다고 가정 - var floatValue float32 - if err := binary.Read(buf, binary.LittleEndian, &floatValue); err != nil { - return nil, fmt.Errorf("failed to read float32: %v", err) - } - floatArray = append(floatArray, floatValue) - } - floatArrays = append(floatArrays, floatArray) - } - - return floatArrays, nil - } - - return nil, nil -} - -/* -func convertUintSliceToIntSlice(uintSlice []uint64) []int64 { - intSlice := make([]int64, len(uintSlice)) - for i, v := range uintSlice { - if v > uint64(^int64(0)) { - return nil - } - intSlice[i] = int64(v) - } - return intSlice -} -*/ // convertInsertMsgToInsertParam는 msgstream.InsertMsg를 api.InsertParam으로 변환합니다. func convertInsertMsgToInsertParam(insertMsg *msgstream.InsertMsg) (*api.InsertParam, error) { if insertMsg == nil { diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 17ec5cd7..575a8e35 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -52,6 +52,7 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat handler.db, err = handler.createDBConnection(handler.connectTimeout) if err != nil { + log.Error("failed to create mysql connection", zap.Error(err)) return nil, err } @@ -305,14 +306,13 @@ func (m *MySQLDataHandler) DropDatabase(ctx context.Context, param *api.DropData return m.mysqlOp(ctx, query) } -func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb.MsgType, msgBytes []byte) error { +func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb.MsgType, dbName string, msgBytes []byte) error { var tsMsg msgstream.TsMsg var err error if msgBytes == nil { log.Warn("msgBytes is nil") return errors.New("msgBytes is nil") - } switch msgType { @@ -337,7 +337,7 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. return err } - tmsg.Database = insertMsg.DbName + tmsg.Database = dbName log.Info("insert msg param", zap.Any("insertMsg", tmsg)) err = m.Insert(ctx, tmsg) if err != nil { @@ -363,7 +363,7 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. return err } - tmsg.Database = deleteMsg.DbName + tmsg.Database = dbName err = m.Delete(ctx, tmsg) if err != nil { @@ -438,7 +438,7 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl } log.Info("replicate message", zap.Any("DbName", param.Database)) - err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), msgBytes) + err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), param.Database, msgBytes) if err != nil { log.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err)) return err diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 3e310d50..013f1269 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -753,7 +753,9 @@ func (e *MetaCDC) startReplicateDMLMsg(replicateCtx context.Context, info *meta. log.Warn("the message pack is nil, the task may be stopping") return } + pChannel := msgPack.EndPositions[0].GetChannelName() + position, targetPosition, err := entity.writerObj.HandleReplicateMessage(replicateCtx, pChannel, msgPack) if err != nil { taskLog.Warn("fail to handle the replicate message", zap.Any("pack", msgPack), zap.Error(err)) From 9ddf0c5b8c6c3fc96a48aea666b7d0b93f433432 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 11:50:17 +0900 Subject: [PATCH 075/118] multi target database --- core/reader/replicate_channel_manager.go | 6 +++--- core/writer/channel_writer.go | 1 + core/writer/mysql_handler.go | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/reader/replicate_channel_manager.go b/core/reader/replicate_channel_manager.go index be9a231b..f8f4fd92 100644 --- a/core/reader/replicate_channel_manager.go +++ b/core/reader/replicate_channel_manager.go @@ -288,7 +288,7 @@ func (r *replicateChannelManager) StartReadCollection(ctx context.Context, info return strings.ToLower(vChannel) } - err = ForeachChannel(targetDBType, info.VirtualChannelNames, targetInfo.VChannels, func(sourceVChannel, targetVChannel string) error { + err = ForeachChannel(targetDBType, info.VirtualChannelNames, targetInfo.VChannels, targetInfo.DatabaseName, func(sourceVChannel, targetVChannel string) error { sourcePChannel := toPhysicalChannel(sourceVChannel) targetPChannel := toPhysicalChannel(targetVChannel) channelHandler, err := r.startReadChannel(&model.SourceCollectionInfo{ @@ -344,7 +344,7 @@ func GetVChannelByPChannel(pChannel string, vChannels []string) string { return "" } -func ForeachChannel(targetDBType string, sourcePChannels, targetPChannels []string, f func(sourcePChannel, targetPChannel string) error) error { +func ForeachChannel(targetDBType string, sourcePChannels, targetPChannels []string, targetDBName string, f func(sourcePChannel, targetPChannel string) error) error { if targetDBType == "milvus" { if len(sourcePChannels) != len(targetPChannels) { return errors.New("the lengths of source and target channels are not equal") @@ -359,7 +359,7 @@ func ForeachChannel(targetDBType string, sourcePChannels, targetPChannels []stri } else { targets = make([]string, len(sourcePChannels)) for i := 0; i < len(sourcePChannels); i++ { - targetPChannels = append(targetPChannels, fmt.Sprintf("%s-dml_%d_v0", targetDBType, i)) + targetPChannels = append(targetPChannels, fmt.Sprintf("%s-%s-dml_%d_v0", targetDBName, targetDBType, i)) } } diff --git a/core/writer/channel_writer.go b/core/writer/channel_writer.go index 3542586d..57372543 100644 --- a/core/writer/channel_writer.go +++ b/core/writer/channel_writer.go @@ -184,6 +184,7 @@ func (c *ChannelWriter) HandleReplicateMessage(ctx context.Context, channelName msgBytesArr = append(msgBytesArr, msgBytes.([]byte)) } log.Info("Message Pack", zap.Any("msgPack", msgPack)) + replicateMessageParam := &api.ReplicateMessageParam{ MsgBaseParam: api.MsgBaseParam{Base: &commonpb.MsgBase{ReplicateInfo: &commonpb.ReplicateInfo{IsReplicate: true}}}, ChannelName: channelName, diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 575a8e35..415c95a5 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -437,6 +437,7 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl return err } + param.Database = strings.Split(param.ChannelName, "-")[0] log.Info("replicate message", zap.Any("DbName", param.Database)) err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), param.Database, msgBytes) if err != nil { From 9d79c325ca38d96727e7bd59fc010f21c1257468 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 12:01:49 +0900 Subject: [PATCH 076/118] multi target database --- core/writer/mysql_handler.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 415c95a5..49069759 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -189,18 +189,17 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e var value, values string for rowCnt := 0; rowCnt < len(rowValues[0]); rowCnt++ { - value = "(" for colNo, _ := range columns { switch rowValues[colNo][rowCnt].(type) { case string: if colNo == 0 { - value = fmt.Sprintf("%s", fmt.Sprintf("%s", rowValues[colNo][rowCnt])) + value = fmt.Sprintf("(%s", fmt.Sprintf("%s", rowValues[colNo][rowCnt])) } else { value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", rowValues[colNo][rowCnt])) } default: if colNo == 0 { - value = fmt.Sprintf("%s", fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + value = fmt.Sprintf("(%s", fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) } else { value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) } From 9f7600b968871bf15e5bd06c486fa97efaa643a3 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 12:14:29 +0900 Subject: [PATCH 077/118] multi target database --- core/writer/mysql_handler.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 49069759..61bef84f 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -85,7 +85,7 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e func (m *MySQLDataHandler) mysqlOp(ctx context.Context, query string, args ...interface{}) error { retryFunc := func() error { log.Info("executing mysql operation", zap.String("query", query), zap.Any("args", args)) - _, err := m.db.ExecContext(ctx, query, args...) + _, err := m.db.Exec(query) if err != nil { log.Warn("failed to execute mysql operation", zap.Error(err)) } @@ -337,7 +337,6 @@ func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb. } tmsg.Database = dbName - log.Info("insert msg param", zap.Any("insertMsg", tmsg)) err = m.Insert(ctx, tmsg) if err != nil { log.Warn("failed to unmarshal insert msg", zap.Error(err)) @@ -437,7 +436,6 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl } param.Database = strings.Split(param.ChannelName, "-")[0] - log.Info("replicate message", zap.Any("DbName", param.Database)) err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), param.Database, msgBytes) if err != nil { log.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err)) From 91df7417657243d61e2404687b981b6e54115103 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 12:25:43 +0900 Subject: [PATCH 078/118] multi target database --- core/writer/channel_writer.go | 1 - core/writer/mysql_handler.go | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/core/writer/channel_writer.go b/core/writer/channel_writer.go index 57372543..15495651 100644 --- a/core/writer/channel_writer.go +++ b/core/writer/channel_writer.go @@ -183,7 +183,6 @@ func (c *ChannelWriter) HandleReplicateMessage(ctx context.Context, channelName } msgBytesArr = append(msgBytesArr, msgBytes.([]byte)) } - log.Info("Message Pack", zap.Any("msgPack", msgPack)) replicateMessageParam := &api.ReplicateMessageParam{ MsgBaseParam: api.MsgBaseParam{Base: &commonpb.MsgBase{ReplicateInfo: &commonpb.ReplicateInfo{IsReplicate: true}}}, diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 61bef84f..1b248275 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -74,11 +74,11 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e } db := sql.OpenDB(connector) - /* - if err := db.Ping(); err != nil { - return nil, fmt.Errorf("failed to ping mysql database : %v", err) - } - */ + + if err := db.Ping(); err != nil { + return nil, fmt.Errorf("failed to ping mysql database : %v", err) + } + return db, nil } From b81c032d0aa27f1e0fefc7df0888f5ff618f215c Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 14:30:57 +0900 Subject: [PATCH 079/118] multi target database --- core/writer/mysql_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 1b248275..3ad3c0a7 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -73,6 +73,7 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e return nil, fmt.Errorf("failed to create connector: %v", err) } + log.Info("connecting to mysql database", zap.Any("connector", connector)) db := sql.OpenDB(connector) if err := db.Ping(); err != nil { From ff0528b7d4b24e6ae7ea90133fb47f004e52e9ec Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 14:59:44 +0900 Subject: [PATCH 080/118] multi target database --- core/writer/mysql_handler.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 3ad3c0a7..e9052b05 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -60,24 +60,18 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat } func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, error) { - connector, err := mysql.NewConnector(&mysql.Config{ + cfg := mysql.Config{ Addr: m.address, User: m.username, Passwd: m.password, Net: "tcp", AllowNativePasswords: true, - DBName: "information_schema", Timeout: time.Duration(connectionTimeout) * time.Second, - }) - if err != nil { - return nil, fmt.Errorf("failed to create connector: %v", err) } - log.Info("connecting to mysql database", zap.Any("connector", connector)) - db := sql.OpenDB(connector) - - if err := db.Ping(); err != nil { - return nil, fmt.Errorf("failed to ping mysql database : %v", err) + db, err := sql.Open("mysql", cfg.FormatDSN()) + if err != nil { + return nil, fmt.Errorf("failed to connect mysql database : %v", err) } return db, nil From 69b7c4a034330e9a4941dab579d57d3bb65a5fe1 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 15:23:55 +0900 Subject: [PATCH 081/118] multi target database --- core/writer/mysql_handler.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index e9052b05..60f0b9b9 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -71,6 +71,7 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e db, err := sql.Open("mysql", cfg.FormatDSN()) if err != nil { + log.Info("failed to connect mysql database", zap.Any("info", cfg.FormatDSN())) return nil, fmt.Errorf("failed to connect mysql database : %v", err) } @@ -161,15 +162,18 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e case *schemapb.FieldData_Vectors: switch vectorData := data.Vectors.Data.(type) { case *schemapb.VectorField_FloatVector: - dim := data.Vectors.Dim - 1 + dim := data.Vectors.Dim + cnt := int64(1) var vec []float32 - for i, v := range vectorData.FloatVector.Data { + for _, v := range vectorData.FloatVector.Data { vec = append(vec, v) - if int64(i)%dim == 0 && i != 0 { + if cnt == dim { colValues = append(colValues, fmt.Sprintf("string_to_vector('[%v]')", join(float32SliceToStringSlice(vec), ","))) vec = []float32{} + cnt = 1 } + cnt++ } rowValues = append(rowValues, colValues) From 665dcd4b9d4f3fedf619e35487b3a17547e20995 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 15:52:43 +0900 Subject: [PATCH 082/118] multi target database --- core/writer/mysql_handler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 60f0b9b9..1fc4cd76 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -172,8 +172,9 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e colValues = append(colValues, fmt.Sprintf("string_to_vector('[%v]')", join(float32SliceToStringSlice(vec), ","))) vec = []float32{} cnt = 1 + } else { + cnt++ } - cnt++ } rowValues = append(rowValues, colValues) From 90e3f4cc87ca1112b290048e23bcc0cefa25b114 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 18:40:48 +0900 Subject: [PATCH 083/118] multi target database --- core/writer/bigquery_handler.go | 124 ++++++++++++-- core/writer/es_handler.go | 278 ++++++++++++++++++++++++++++++++ core/writer/mysql_handler.go | 1 - 3 files changed, 388 insertions(+), 15 deletions(-) create mode 100644 core/writer/es_handler.go diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index a2b193cf..e3531742 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -3,6 +3,7 @@ package writer import ( "context" "fmt" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "time" "cloud.google.com/go/bigquery" @@ -113,27 +114,122 @@ func (m *BigQueryDataHandler) DropTable(ctx context.Context, param *api.DropColl } func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { - inserter := m.client.Dataset(param.Database).Table(param.CollectionName).Inserter() - var rows []*bigquery.ValuesSaver - var colName bigquery.Schema - var colValues []bigquery.Value - - for i, col := range param.Columns { - colName[i].Name = col.Name() - colValues[i] = col.FieldData() + columns := []string{} + + rowValues := [][]interface{}{} + for _, col := range param.Columns { + columns = append(columns, fmt.Sprintf("`%s`", col.Name())) + + colValues := []interface{}{} + switch data := col.FieldData().GetField().(type) { + case *schemapb.FieldData_Scalars: + switch scalarData := data.Scalars.Data.(type) { + case *schemapb.ScalarField_LongData: + for _, v := range scalarData.LongData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_BoolData: + for _, v := range scalarData.BoolData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_StringData: + for _, v := range scalarData.StringData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + case *schemapb.ScalarField_ArrayData: + for _, v := range scalarData.ArrayData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + case *schemapb.ScalarField_IntData: + for _, v := range scalarData.IntData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_FloatData: + for _, v := range scalarData.FloatData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_DoubleData: + for _, v := range scalarData.DoubleData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_JsonData: + for _, v := range scalarData.JsonData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + case *schemapb.ScalarField_BytesData: + for _, v := range scalarData.BytesData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + default: + return fmt.Errorf("unsupported scalar data type: %T", scalarData) + } + + rowValues = append(rowValues, colValues) + case *schemapb.FieldData_Vectors: + switch vectorData := data.Vectors.Data.(type) { + case *schemapb.VectorField_FloatVector: + dim := data.Vectors.Dim + cnt := int64(1) + var vec []float32 + for _, v := range vectorData.FloatVector.Data { + vec = append(vec, v) + + if cnt == dim { + colValues = append(colValues, fmt.Sprintf("[%v]", join(float32SliceToStringSlice(vec), ","))) + vec = []float32{} + cnt = 1 + } else { + cnt++ + } + } + + rowValues = append(rowValues, colValues) + default: + return fmt.Errorf("unsupported vector data type: %T", vectorData) + } + default: + return fmt.Errorf("unsupported field data type: %T", data) + } } - row := &bigquery.ValuesSaver{ - Schema: colName, - Row: colValues, + var value, values string + + for rowCnt := 0; rowCnt < len(rowValues[0]); rowCnt++ { + for colNo, _ := range columns { + switch rowValues[colNo][rowCnt].(type) { + case string: + if colNo == 0 { + value = fmt.Sprintf("(%s", fmt.Sprintf("%s", rowValues[colNo][rowCnt])) + } else { + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", rowValues[colNo][rowCnt])) + } + default: + if colNo == 0 { + value = fmt.Sprintf("(%s", fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + } else { + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + } + } + } + value = fmt.Sprintf("%s)", value) + + if rowCnt == 0 { + values = fmt.Sprintf("%s", value) + } else { + values = fmt.Sprintf("%s,%s", values, value) + } } - rows = append(rows, row) - return inserter.Put(ctx, rows) + //string_to_vector('[1,2,3]') + query := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s", + param.Database, param.CollectionName, join(columns, ","), values) + // log.Info("INSERT", zap.String("query", query)) + + return m.bigqueryOp(ctx, query, nil) } func (m *BigQueryDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { - query := fmt.Sprintf("DELETE FROM `%s.%s` WHERE %s = @value", param.Database, param.CollectionName, param.Column.Name()) + query := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE `%s` = @value", param.Database, param.CollectionName, param.Column.Name()) params := map[string]interface{}{"value": param.Column.FieldData()} return m.bigqueryOp(ctx, query, params) } diff --git a/core/writer/es_handler.go b/core/writer/es_handler.go new file mode 100644 index 00000000..6d9449e3 --- /dev/null +++ b/core/writer/es_handler.go @@ -0,0 +1,278 @@ +package writer + +import ( + "context" + "fmt" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "time" + + "cloud.google.com/go/bigquery" + "github.com/cenkalti/backoff/v4" + "github.com/cockroachdb/errors" + "go.uber.org/zap" + "google.golang.org/api/option" + + "github.com/zilliztech/milvus-cdc/core/api" + "github.com/zilliztech/milvus-cdc/core/config" + "github.com/zilliztech/milvus-cdc/core/log" +) + +type ESDataHandler struct { + api.DataHandler + + projectID string + credentials string + connectTimeout int + client *bigquery.Client + retryOptions *backoff.ExponentialBackOff +} + +// NewBigQueryDataHandler options must include ProjectIDOption and CredentialsOption +func NewESDataHandler(options ...config.Option[*ESDataHandler]) (*ESDataHandler, error) { + handler := &ESDataHandler{ + connectTimeout: 5, + retryOptions: backoff.NewExponentialBackOff(), + } + handler.retryOptions.MaxElapsedTime = 2 * time.Minute // 설정된 최대 재시도 시간 + + for _, option := range options { + option.Apply(handler) + } + if handler.projectID == "" { + return nil, errors.New("empty BigQuery project ID") + } + if handler.credentials == "" { + return nil, errors.New("empty BigQuery credentials") + } + + var err error + timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) + defer cancel() + + handler.client, err = handler.createESClient(timeoutContext) + if err != nil { + return nil, err + } + + return handler, nil +} + +func (m *ESDataHandler) createESClient(ctx context.Context) (*bigquery.Client, error) { + client, err := bigquery.NewClient(ctx, m.projectID, option.WithCredentialsFile(m.credentials)) + if err != nil { + return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + } + return client, nil +} + +func (m *ESDataHandler) ESOp(ctx context.Context, query string, params map[string]interface{}) error { + retryFunc := func() error { + q := m.client.Query(query) + q.Parameters = make([]bigquery.QueryParameter, 0, len(params)) + for k, v := range params { + q.Parameters = append(q.Parameters, bigquery.QueryParameter{Name: k, Value: v}) + } + job, err := q.Run(ctx) + if err != nil { + return err + } + status, err := job.Wait(ctx) + if err != nil { + return err + } + if err := status.Err(); err != nil { + return err + } + return nil + } + + err := backoff.Retry(retryFunc, backoff.WithContext(m.retryOptions, ctx)) + if err != nil { + log.Warn("retry operation failed", zap.Error(err)) + } + return err +} + +func (m *ESDataHandler) CreateTable(ctx context.Context, param *api.CreateCollectionParam) error { + schema := bigquery.Schema{} + for _, field := range param.Schema.Fields { + schema = append(schema, &bigquery.FieldSchema{ + Name: field.Name, + Type: bigquery.FieldType(field.DataType), + }) + } + metaData := &bigquery.TableMetadata{ + Schema: schema, + } + tableRef := m.client.Dataset(param.Database).Table(param.Schema.CollectionName) + return tableRef.Create(ctx, metaData) +} + +func (m *ESDataHandler) DropTable(ctx context.Context, param *api.DropCollectionParam) error { + tableRef := m.client.Dataset(param.Database).Table(param.CollectionName) + return tableRef.Delete(ctx) +} + +func (m *ESDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { + columns := []string{} + + rowValues := [][]interface{}{} + for _, col := range param.Columns { + columns = append(columns, fmt.Sprintf("`%s`", col.Name())) + + colValues := []interface{}{} + switch data := col.FieldData().GetField().(type) { + case *schemapb.FieldData_Scalars: + switch scalarData := data.Scalars.Data.(type) { + case *schemapb.ScalarField_LongData: + for _, v := range scalarData.LongData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_BoolData: + for _, v := range scalarData.BoolData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_StringData: + for _, v := range scalarData.StringData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + case *schemapb.ScalarField_ArrayData: + for _, v := range scalarData.ArrayData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + case *schemapb.ScalarField_IntData: + for _, v := range scalarData.IntData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_FloatData: + for _, v := range scalarData.FloatData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_DoubleData: + for _, v := range scalarData.DoubleData.Data { + colValues = append(colValues, v) + } + case *schemapb.ScalarField_JsonData: + for _, v := range scalarData.JsonData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + case *schemapb.ScalarField_BytesData: + for _, v := range scalarData.BytesData.Data { + colValues = append(colValues, fmt.Sprintf("'%s'", v)) + } + default: + return fmt.Errorf("unsupported scalar data type: %T", scalarData) + } + + rowValues = append(rowValues, colValues) + case *schemapb.FieldData_Vectors: + switch vectorData := data.Vectors.Data.(type) { + case *schemapb.VectorField_FloatVector: + dim := data.Vectors.Dim + cnt := int64(1) + var vec []float32 + for _, v := range vectorData.FloatVector.Data { + vec = append(vec, v) + + if cnt == dim { + colValues = append(colValues, fmt.Sprintf("string_to_vector('[%v]')", join(float32SliceToStringSlice(vec), ","))) + vec = []float32{} + cnt = 1 + } else { + cnt++ + } + } + + rowValues = append(rowValues, colValues) + default: + return fmt.Errorf("unsupported vector data type: %T", vectorData) + } + default: + return fmt.Errorf("unsupported field data type: %T", data) + } + } + + var value, values string + + for rowCnt := 0; rowCnt < len(rowValues[0]); rowCnt++ { + for colNo, _ := range columns { + switch rowValues[colNo][rowCnt].(type) { + case string: + if colNo == 0 { + value = fmt.Sprintf("(%s", fmt.Sprintf("%s", rowValues[colNo][rowCnt])) + } else { + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", rowValues[colNo][rowCnt])) + } + default: + if colNo == 0 { + value = fmt.Sprintf("(%s", fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + } else { + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + } + } + } + value = fmt.Sprintf("%s)", value) + + if rowCnt == 0 { + values = fmt.Sprintf("%s", value) + } else { + values = fmt.Sprintf("%s,%s", values, value) + } + } + + //string_to_vector('[1,2,3]') + query := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s", + param.Database, param.CollectionName, join(columns, ","), values) + // log.Info("INSERT", zap.String("query", query)) + + return m.ESOp(ctx, query, nil) +} + +func (m *ESDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { + query := fmt.Sprintf("DELETE FROM `%s.%s` WHERE %s = @value", param.Database, param.CollectionName, param.Column.Name()) + params := map[string]interface{}{"value": param.Column.FieldData()} + return m.ESOp(ctx, query, params) +} + +func (m *ESDataHandler) CreateIndex(ctx context.Context, param *api.CreateIndexParam) error { + // BigQuery는 Vector Search Index만을 지원합니다. + query := fmt.Sprintf("CREATE OR REPLACE VECTOR INDEX `%s`.%s ON `%s`(embedding_v1) OPTIONS(distance_type='L2',index_type_type='IVF')", param.Database, param.IndexName, param.CollectionName) + return m.ESOp(ctx, query, nil) +} + +func (m *ESDataHandler) DropIndex(ctx context.Context, param *api.DropIndexParam) error { + // BigQuery는 인덱스를 지원하지 않습니다. + return nil +} + +func (m *ESDataHandler) CreateDatabase(ctx context.Context, param *api.CreateDatabaseParam) error { + // BigQuery에서는 데이터셋을 생성하는 방식입니다. + dataset := m.client.Dataset(param.DbName) + return dataset.Create(ctx, &bigquery.DatasetMetadata{}) +} + +func (m *ESDataHandler) DropDatabase(ctx context.Context, param *api.DropDatabaseParam) error { + // BigQuery에서는 데이터셋을 삭제하는 방식입니다. + dataset := m.client.Dataset(param.DbName) + return dataset.Delete(ctx) +} + +func (m *ESDataHandler) DescribeTable(ctx context.Context, param *api.DescribeCollectionParam) error { + tableRef := m.client.Dataset(param.Database).Table(param.Name) + metaData, err := tableRef.Metadata(ctx) + if err != nil { + return err + } + fmt.Printf("Table: %s, Schema: %v\n", param.Name, metaData.Schema) + return nil +} + +func (m *ESDataHandler) DescribeDatabase(ctx context.Context, param *api.DescribeDatabaseParam) error { + dataset := m.client.Dataset(param.Name) + metaData, err := dataset.Metadata(ctx) + if err != nil { + return err + } + fmt.Printf("Dataset: %s, Location: %s\n", param.Name, metaData.Location) + return nil +} diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 1fc4cd76..1b1a34f6 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -220,7 +220,6 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e // log.Info("INSERT", zap.String("query", query)) return m.mysqlOp(ctx, query) - } func interfaceSliceToStringSlice(input []interface{}) []string { From 041d6224bd0131e0487d486cd7493290696ad775 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 19:10:40 +0900 Subject: [PATCH 084/118] multi target database --- core/writer/bigquery_handler.go | 6 +- core/writer/config_option.go | 21 +++++ core/writer/es_handler.go | 136 ++++++++++++++++---------------- server/cdc_impl.go | 33 +++++--- 4 files changed, 114 insertions(+), 82 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index e3531742..dc13a4f1 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -93,7 +93,7 @@ func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, para return err } -func (m *BigQueryDataHandler) CreateTable(ctx context.Context, param *api.CreateCollectionParam) error { +func (m *BigQueryDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { schema := bigquery.Schema{} for _, field := range param.Schema.Fields { schema = append(schema, &bigquery.FieldSchema{ @@ -108,7 +108,7 @@ func (m *BigQueryDataHandler) CreateTable(ctx context.Context, param *api.Create return tableRef.Create(ctx, metaData) } -func (m *BigQueryDataHandler) DropTable(ctx context.Context, param *api.DropCollectionParam) error { +func (m *BigQueryDataHandler) DropCollection(ctx context.Context, param *api.DropCollectionParam) error { tableRef := m.client.Dataset(param.Database).Table(param.CollectionName) return tableRef.Delete(ctx) } @@ -257,7 +257,7 @@ func (m *BigQueryDataHandler) DropDatabase(ctx context.Context, param *api.DropD return dataset.Delete(ctx) } -func (m *BigQueryDataHandler) DescribeTable(ctx context.Context, param *api.DescribeCollectionParam) error { +func (m *BigQueryDataHandler) DescribeCollection(ctx context.Context, param *api.DescribeCollectionParam) error { tableRef := m.client.Dataset(param.Database).Table(param.Name) metaData, err := tableRef.Metadata(ctx) if err != nil { diff --git a/core/writer/config_option.go b/core/writer/config_option.go index 8053edca..b3add3e2 100644 --- a/core/writer/config_option.go +++ b/core/writer/config_option.go @@ -129,3 +129,24 @@ func BigQueryConnectTimeoutOption(timeout int) config.Option[*BigQueryDataHandle } }) } + +func ESUserOption(username string, password string) config.Option[*ESDataHandler] { + return config.OptionFunc[*ESDataHandler](func(object *ESDataHandler) { + object.username = username + object.password = password + }) +} + +func ESAddressOption(address string) config.Option[*ESDataHandler] { + return config.OptionFunc[*ESDataHandler](func(object *ESDataHandler) { + object.address = address + }) +} + +func ESConnectTimeoutOption(timeout int) config.Option[*ESDataHandler] { + return config.OptionFunc[*ESDataHandler](func(object *ESDataHandler) { + if timeout > 0 { + object.connectTimeout = timeout + } + }) +} diff --git a/core/writer/es_handler.go b/core/writer/es_handler.go index 6d9449e3..682ad26c 100644 --- a/core/writer/es_handler.go +++ b/core/writer/es_handler.go @@ -2,29 +2,30 @@ package writer import ( "context" + "database/sql" "fmt" + "github.com/go-sql-driver/mysql" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "time" - "cloud.google.com/go/bigquery" "github.com/cenkalti/backoff/v4" "github.com/cockroachdb/errors" - "go.uber.org/zap" - "google.golang.org/api/option" - "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" + "go.uber.org/zap" ) type ESDataHandler struct { api.DataHandler - projectID string - credentials string + address string + username string + password string connectTimeout int - client *bigquery.Client + db *sql.DB retryOptions *backoff.ExponentialBackOff + mysqlCli *sql.DB } // NewBigQueryDataHandler options must include ProjectIDOption and CredentialsOption @@ -38,52 +39,49 @@ func NewESDataHandler(options ...config.Option[*ESDataHandler]) (*ESDataHandler, for _, option := range options { option.Apply(handler) } - if handler.projectID == "" { - return nil, errors.New("empty BigQuery project ID") - } - if handler.credentials == "" { - return nil, errors.New("empty BigQuery credentials") + if handler.address == "" { + return nil, errors.New("empty MySQL address") } var err error - timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) - defer cancel() - handler.client, err = handler.createESClient(timeoutContext) + handler.db, err = handler.createESClient(handler.connectTimeout) if err != nil { + log.Error("failed to create mysql connection", zap.Error(err)) return nil, err } return handler, nil } -func (m *ESDataHandler) createESClient(ctx context.Context) (*bigquery.Client, error) { - client, err := bigquery.NewClient(ctx, m.projectID, option.WithCredentialsFile(m.credentials)) +func (m *ESDataHandler) createESClient(connectionTimeout int) (*sql.DB, error) { + cfg := mysql.Config{ + Addr: m.address, + User: m.username, + Passwd: m.password, + Net: "tcp", + AllowNativePasswords: true, + Timeout: time.Duration(connectionTimeout) * time.Second, + } + + db, err := sql.Open("mysql", cfg.FormatDSN()) if err != nil { - return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + log.Info("failed to connect mysql database", zap.Any("info", cfg.FormatDSN())) + return nil, fmt.Errorf("failed to connect mysql database : %v", err) } - return client, nil + + return db, nil + } -func (m *ESDataHandler) ESOp(ctx context.Context, query string, params map[string]interface{}) error { +func (m *ESDataHandler) ESOp(ctx context.Context, query string, args ...interface{}) error { retryFunc := func() error { - q := m.client.Query(query) - q.Parameters = make([]bigquery.QueryParameter, 0, len(params)) - for k, v := range params { - q.Parameters = append(q.Parameters, bigquery.QueryParameter{Name: k, Value: v}) - } - job, err := q.Run(ctx) - if err != nil { - return err - } - status, err := job.Wait(ctx) + log.Info("executing mysql operation", zap.String("query", query), zap.Any("args", args)) + _, err := m.db.Exec(query) if err != nil { - return err - } - if err := status.Err(); err != nil { - return err + log.Warn("failed to execute mysql operation", zap.Error(err)) } - return nil + return err } err := backoff.Retry(retryFunc, backoff.WithContext(m.retryOptions, ctx)) @@ -93,24 +91,16 @@ func (m *ESDataHandler) ESOp(ctx context.Context, query string, params map[strin return err } -func (m *ESDataHandler) CreateTable(ctx context.Context, param *api.CreateCollectionParam) error { - schema := bigquery.Schema{} - for _, field := range param.Schema.Fields { - schema = append(schema, &bigquery.FieldSchema{ - Name: field.Name, - Type: bigquery.FieldType(field.DataType), - }) - } - metaData := &bigquery.TableMetadata{ - Schema: schema, - } - tableRef := m.client.Dataset(param.Database).Table(param.Schema.CollectionName) - return tableRef.Create(ctx, metaData) +func (m *ESDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { + query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", param.Schema.CollectionName, param.Schema.Fields) + log.Info("CREATE TABLE", zap.String("query", query)) + return m.ESOp(ctx, query) } -func (m *ESDataHandler) DropTable(ctx context.Context, param *api.DropCollectionParam) error { - tableRef := m.client.Dataset(param.Database).Table(param.CollectionName) - return tableRef.Delete(ctx) +func (m *ESDataHandler) DropCollection(ctx context.Context, param *api.DropCollectionParam) error { + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", param.CollectionName) + log.Info("DROP TABLE", zap.String("query", query)) + return m.ESOp(ctx, query) } func (m *ESDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { @@ -246,33 +236,41 @@ func (m *ESDataHandler) DropIndex(ctx context.Context, param *api.DropIndexParam } func (m *ESDataHandler) CreateDatabase(ctx context.Context, param *api.CreateDatabaseParam) error { - // BigQuery에서는 데이터셋을 생성하는 방식입니다. - dataset := m.client.Dataset(param.DbName) - return dataset.Create(ctx, &bigquery.DatasetMetadata{}) + query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", param.DbName) + log.Info("CREATE DATABASE", zap.String("query", query)) + return m.ESOp(ctx, query) } func (m *ESDataHandler) DropDatabase(ctx context.Context, param *api.DropDatabaseParam) error { - // BigQuery에서는 데이터셋을 삭제하는 방식입니다. - dataset := m.client.Dataset(param.DbName) - return dataset.Delete(ctx) + query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", param.DbName) + log.Info("DROP DATABASE", zap.String("query", query)) + return m.ESOp(ctx, query) } -func (m *ESDataHandler) DescribeTable(ctx context.Context, param *api.DescribeCollectionParam) error { - tableRef := m.client.Dataset(param.Database).Table(param.Name) - metaData, err := tableRef.Metadata(ctx) - if err != nil { - return err - } - fmt.Printf("Table: %s, Schema: %v\n", param.Name, metaData.Schema) - return nil +func (m *ESDataHandler) DescribeCollection(ctx context.Context, param *api.DescribeCollectionParam) error { + query := fmt.Sprintf("DESCRIBE %s", param.Name) + log.Info("DESCRIBE", zap.String("query", query)) + return m.ESOp(ctx, query) } func (m *ESDataHandler) DescribeDatabase(ctx context.Context, param *api.DescribeDatabaseParam) error { - dataset := m.client.Dataset(param.Name) - metaData, err := dataset.Metadata(ctx) + query := "SHOW DATABASES" + log.Info(query) + rows, err := m.db.QueryContext(ctx, query) if err != nil { return err } - fmt.Printf("Dataset: %s, Location: %s\n", param.Name, metaData.Location) - return nil + defer rows.Close() + + var dbName string + for rows.Next() { + err := rows.Scan(&dbName) + if err != nil { + return err + } + if dbName == param.Name { + return nil + } + } + return errors.Newf("database [%s] not found", param.Name) } diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 013f1269..292f6ebd 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -303,18 +303,21 @@ func (e *MetaCDC) Create(req *request.CreateRequest) (resp *request.CreateRespon func (e *MetaCDC) validCreateRequest(req *request.CreateRequest) error { connectParam := req.MilvusConnectParam - if connectParam.Host == "" { - return servererror.NewClientError("the milvus host is empty") - } - if connectParam.Port <= 0 { - return servererror.NewClientError("the milvus port is less or equal zero") - } - if (connectParam.Username != "" && connectParam.Password == "") || - (connectParam.Username == "" && connectParam.Password != "") { - return servererror.NewClientError("cannot set only one of the milvus username and password") + if connectParam.TargetDBType != "bigquery" { + if connectParam.Host == "" { + return servererror.NewClientError("the " + connectParam.TargetDBType + " host is empty") + } + if connectParam.Port <= 0 { + return servererror.NewClientError("the " + connectParam.TargetDBType + " port is less or equal zero") + } + if (connectParam.Username != "" && connectParam.Password == "") || + (connectParam.Username == "" && connectParam.Password != "") { + return servererror.NewClientError("cannot set only one of the " + connectParam.TargetDBType + " username and password") + } } + if connectParam.ConnectTimeout < 0 { - return servererror.NewClientError("the milvus connect timeout is less zero") + return servererror.NewClientError("the " + connectParam.TargetDBType + " connect timeout is less zero") } cacheParam := req.BufferConfig if cacheParam.Period < 0 { @@ -360,6 +363,15 @@ func (e *MetaCDC) validCreateRequest(req *request.CreateRequest) error { log.Warn("fail to connect the mysql", zap.Any("connect_param", connectParam), zap.Error(err)) return errors.WithMessage(err, "fail to connect the mysql") } + } else if strings.ToLower(connectParam.TargetDBType) == "elasticsearch" { + _, err = cdcwriter.NewESDataHandler( + cdcwriter.ESAddressOption(fmt.Sprintf("%s:%d", connectParam.Host, connectParam.Port)), + cdcwriter.ESUserOption(connectParam.Username, connectParam.Password), + cdcwriter.ESConnectTimeoutOption(connectParam.ConnectTimeout)) + if err != nil { + log.Warn("fail to connect the mysql", zap.Any("connect_param", connectParam), zap.Error(err)) + return errors.WithMessage(err, "fail to connect the mysql") + } } return nil @@ -641,6 +653,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err MessageBufferSize: bufferSize, Retry: e.config.Retry, }, metaOp.GetAllDroppedObj()) + } else if strings.ToLower(targetConfig.TargetDBType) == "elasticsearch" { } e.replicateEntityMap.Lock() From 1235d9174aaaa5364470a40cf48fca25de420ab1 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 19:18:47 +0900 Subject: [PATCH 085/118] multi target database --- core/writer/bigquery_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index dc13a4f1..f46531fe 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -42,7 +42,7 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi return nil, errors.New("empty BigQuery project ID") } if handler.credentials == "" { - return nil, errors.New("empty BigQuery credentials") + log.Warn("empty BigQuery credentials") } var err error From 8b383772a8aa8a397d4e1268a0083e2aa7fa0208 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 19:37:51 +0900 Subject: [PATCH 086/118] multi target database --- core/writer/bigquery_handler.go | 146 ++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index f46531fe..da8de726 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -3,7 +3,10 @@ package writer import ( "context" "fmt" + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/pkg/mq/msgstream" + "strings" "time" "cloud.google.com/go/bigquery" @@ -12,6 +15,7 @@ import ( "go.uber.org/zap" "google.golang.org/api/option" + "github.com/golang/protobuf/proto" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" @@ -257,6 +261,148 @@ func (m *BigQueryDataHandler) DropDatabase(ctx context.Context, param *api.DropD return dataset.Delete(ctx) } +func (m *BigQueryDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb.MsgType, dbName string, msgBytes []byte) error { + var tsMsg msgstream.TsMsg + var err error + + if msgBytes == nil { + log.Warn("msgBytes is nil") + return errors.New("msgBytes is nil") + } + + switch msgType { + case commonpb.MsgType_Insert: + tsMsg = &msgstream.InsertMsg{} + + msg, err := tsMsg.Unmarshal(msgBytes) + if err != nil { + log.Warn("failed to unmarshal ts msg", zap.Error(err)) + return err + } + + log.Info("unmarshalTsMsg", zap.Any("msgType", msgType), zap.Any("msg", msg)) + + insertMsg := msg.(*msgstream.InsertMsg) + + log.Info("insert msg", zap.Any("insertMsg", insertMsg.InsertRequest), zap.Any("insertMsgRows", insertMsg.NumRows)) + log.Info("msgBytes", zap.Any("msgBytes", msgBytes)) + tmsg, err := convertInsertMsgToInsertParam(insertMsg) + if err != nil { + log.Warn("failed to convert insert msg to insert param", zap.Error(err)) + return err + } + + tmsg.Database = dbName + err = m.Insert(ctx, tmsg) + if err != nil { + log.Warn("failed to unmarshal insert msg", zap.Error(err)) + return err + } + case commonpb.MsgType_Delete: + tsMsg = &msgstream.DeleteMsg{} + + msg, err := tsMsg.Unmarshal(msgBytes) + if err != nil { + log.Warn("failed to unmarshal ts msg", zap.Error(err)) + return err + } + + log.Info("unmarshalTsMsg", zap.Any("msgType", msgType), zap.Any("msg", msg)) + + deleteMsg := msg.(*msgstream.DeleteMsg) + + tmsg, err := convertDeleteMsgToDeleteParam(deleteMsg) + if err != nil { + log.Warn("failed to convert delete msg to delete param", zap.Error(err)) + return err + } + + tmsg.Database = dbName + + err = m.Delete(ctx, tmsg) + if err != nil { + log.Warn("failed to delete", zap.Error(err)) + return err + } + case commonpb.MsgType_Upsert: + // UpsertMsg는 InsertMsg와 DeleteMsg를 포함하므로, 따로 언마샬링한 후 조립합니다. + // tsMsg = &msgstream.UpsertMsg{} + insertMsg := &msgstream.InsertMsg{} + deleteMsg := &msgstream.DeleteMsg{} + + // msgBytes를 분할하여 각 메시지에 언마샬링합니다. (여기서는 단순히 msgBytes를 나눠서 가정합니다.) + half := len(msgBytes) / 2 + err = proto.Unmarshal(msgBytes[:half], insertMsg) + if err != nil { + log.Warn("failed to unmarshal insert msg", zap.Error(err)) + return err + } + + imsg, err := convertInsertMsgToInsertParam(insertMsg) + if err != nil { + log.Warn("failed to convert insert msg to insert param", zap.Error(err)) + return err + } + + err = m.Insert(ctx, imsg) + if err != nil { + log.Warn("failed to insert", zap.Error(err)) + return err + } + + err = proto.Unmarshal(msgBytes[half:], deleteMsg) + if err != nil { + return err + } + + dmsg, err := convertDeleteMsgToDeleteParam(deleteMsg) + if err != nil { + log.Warn("failed to convert delete msg to delete param", zap.Error(err)) + return err + } + + err = m.Delete(ctx, dmsg) + if err != nil { + log.Warn("failed to delete", zap.Error(err)) + return err + } + case commonpb.MsgType_TimeTick: + return nil + default: + log.Warn("unsupported message type", zap.Any("msgType", msgType)) + err = fmt.Errorf("unsupported message type: %v", msgType) + return err + } + + return nil +} + +func (m *BigQueryDataHandler) ReplicateMessage(ctx context.Context, param *api.ReplicateMessageParam) error { + for i, msgBytes := range param.MsgsBytes { + header := &commonpb.MsgHeader{} + err := proto.Unmarshal(msgBytes, header) + if err != nil { + log.Warn("failed to unmarshal msg header", zap.Int("index", i), zap.Error(err)) + return err + } + + if header.GetBase() == nil { + log.Warn("msg header base is nil", zap.Int("index", i)) + return err + } + + param.Database = strings.Split(param.ChannelName, "-")[0] + err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), param.Database, msgBytes) + if err != nil { + log.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err)) + return err + } + + } + + return nil +} + func (m *BigQueryDataHandler) DescribeCollection(ctx context.Context, param *api.DescribeCollectionParam) error { tableRef := m.client.Dataset(param.Database).Table(param.Name) metaData, err := tableRef.Metadata(ctx) From e6b4507a78e37e12d201d8b5116055c0aac33331 Mon Sep 17 00:00:00 2001 From: ianlee Date: Tue, 30 Jul 2024 19:59:42 +0900 Subject: [PATCH 087/118] multi target database --- core/writer/bigquery_handler.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index da8de726..af587729 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -227,7 +227,7 @@ func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam //string_to_vector('[1,2,3]') query := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s", param.Database, param.CollectionName, join(columns, ","), values) - // log.Info("INSERT", zap.String("query", query)) + log.Info("INSERT", zap.String("query", query)) return m.bigqueryOp(ctx, query, nil) } @@ -284,8 +284,6 @@ func (m *BigQueryDataHandler) unmarshalTsMsg(ctx context.Context, msgType common insertMsg := msg.(*msgstream.InsertMsg) - log.Info("insert msg", zap.Any("insertMsg", insertMsg.InsertRequest), zap.Any("insertMsgRows", insertMsg.NumRows)) - log.Info("msgBytes", zap.Any("msgBytes", msgBytes)) tmsg, err := convertInsertMsgToInsertParam(insertMsg) if err != nil { log.Warn("failed to convert insert msg to insert param", zap.Error(err)) From 4c84a6d08e99c665868537958db5b64ab3d50ddb Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 11:22:33 +0900 Subject: [PATCH 088/118] multi target database --- core/writer/bigquery_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index af587729..a6da16a5 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -78,6 +78,7 @@ func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, para } job, err := q.Run(ctx) if err != nil { + log.Warn("failed to run query", zap.Error(err)) return err } status, err := job.Wait(ctx) From c5dce05d56b4c38ad5f47590b08806bf15ee530f Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 11:45:36 +0900 Subject: [PATCH 089/118] multi target database --- core/writer/bigquery_handler.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index a6da16a5..1b90fe22 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -12,13 +12,11 @@ import ( "cloud.google.com/go/bigquery" "github.com/cenkalti/backoff/v4" "github.com/cockroachdb/errors" - "go.uber.org/zap" - "google.golang.org/api/option" - "github.com/golang/protobuf/proto" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" + "go.uber.org/zap" ) type BigQueryDataHandler struct { @@ -62,7 +60,7 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi } func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - client, err := bigquery.NewClient(ctx, m.projectID, option.WithCredentialsFile(m.credentials)) + client, err := bigquery.NewClient(ctx, m.projectID) if err != nil { return nil, fmt.Errorf("failed to create BigQuery client: %v", err) } From 0e0f24bf4646f089e5be931ae6c401d8fa2d4659 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 11:57:13 +0900 Subject: [PATCH 090/118] multi target database --- core/writer/bigquery_handler.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 1b90fe22..591f3e21 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -2,10 +2,13 @@ package writer import ( "context" + "crypto/tls" "fmt" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" + "google.golang.org/api/option" + "net/http" "strings" "time" @@ -59,6 +62,32 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi return handler, nil } +func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { + // TLS 인증서 검증을 비활성화하는 HTTP 클라이언트 설정 + insecureTransport := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + httpClient := &http.Client{ + Transport: insecureTransport, + } + + // 클라이언트 옵션 설정 + clientOptions := []option.ClientOption{ + option.WithHTTPClient(httpClient), + } + + // BigQuery 클라이언트 생성 + client, err := bigquery.NewClient(ctx, m.projectID, clientOptions...) + if err != nil { + return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + } + + return client, nil +} + +/* func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { client, err := bigquery.NewClient(ctx, m.projectID) if err != nil { @@ -66,6 +95,7 @@ func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigque } return client, nil } +*/ func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { retryFunc := func() error { From f212e506eadf83977de5c099c8988952aab2c7f1 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 15:08:30 +0900 Subject: [PATCH 091/118] multi target database --- core/writer/bigquery_handler.go | 38 +++++++-------------------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 591f3e21..40838fc1 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -1,25 +1,26 @@ package writer import ( + secretmanager "cloud.google.com/go/secretmanager/apiv1" "context" - "crypto/tls" "fmt" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "google.golang.org/api/option" - "net/http" + "golang.org/x/oauth2/google" "strings" "time" "cloud.google.com/go/bigquery" "github.com/cenkalti/backoff/v4" "github.com/cockroachdb/errors" + "go.uber.org/zap" + "google.golang.org/api/option" + "github.com/golang/protobuf/proto" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" - "go.uber.org/zap" ) type BigQueryDataHandler struct { @@ -63,39 +64,16 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi } func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - // TLS 인증서 검증을 비활성화하는 HTTP 클라이언트 설정 - insecureTransport := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - httpClient := &http.Client{ - Transport: insecureTransport, - } - - // 클라이언트 옵션 설정 - clientOptions := []option.ClientOption{ - option.WithHTTPClient(httpClient), - } - - // BigQuery 클라이언트 생성 - client, err := bigquery.NewClient(ctx, m.projectID, clientOptions...) + creds, err := google.CredentialsFromJSON(ctx, []byte("JSON creds"), secretmanager.DefaultAuthScopes()...) if err != nil { - return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + // TODO: handle error. } - - return client, nil -} - -/* -func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - client, err := bigquery.NewClient(ctx, m.projectID) + client, err := bigquery.NewClient(ctx, m.projectID, option.WithCredentials(creds)) if err != nil { return nil, fmt.Errorf("failed to create BigQuery client: %v", err) } return client, nil } -*/ func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { retryFunc := func() error { From 1e9f0b77b75c2b7d90a31b234747b3ac128cc1fc Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 15:13:54 +0900 Subject: [PATCH 092/118] multi target database --- core/writer/bigquery_handler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 40838fc1..379653fa 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -66,10 +66,12 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { creds, err := google.CredentialsFromJSON(ctx, []byte("JSON creds"), secretmanager.DefaultAuthScopes()...) if err != nil { - // TODO: handle error. + log.Warn("failed to get crediential of client", zap.Error(err)) + return nil, fmt.Errorf("failed to get crediential of client: %v", err) } client, err := bigquery.NewClient(ctx, m.projectID, option.WithCredentials(creds)) if err != nil { + log.Warn("failed to create BigQuery client", zap.Error(err)) return nil, fmt.Errorf("failed to create BigQuery client: %v", err) } return client, nil From ffac7b637804d79fb98fa75253192b1e031aab16 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 15:40:43 +0900 Subject: [PATCH 093/118] multi target database --- core/writer/bigquery_handler.go | 42 ++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 379653fa..a4dea35c 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -1,13 +1,13 @@ package writer import ( - secretmanager "cloud.google.com/go/secretmanager/apiv1" "context" + "crypto/tls" "fmt" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "golang.org/x/oauth2/google" + "net/http" "strings" "time" @@ -64,19 +64,45 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi } func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - creds, err := google.CredentialsFromJSON(ctx, []byte("JSON creds"), secretmanager.DefaultAuthScopes()...) - if err != nil { - log.Warn("failed to get crediential of client", zap.Error(err)) - return nil, fmt.Errorf("failed to get crediential of client: %v", err) + // TLS 인증서 검증을 비활성화하는 HTTP 클라이언트 설정 + insecureTransport := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + httpClient := &http.Client{ + Transport: insecureTransport, + } + + // 클라이언트 옵션 설정 + clientOptions := []option.ClientOption{ + option.WithHTTPClient(httpClient), } - client, err := bigquery.NewClient(ctx, m.projectID, option.WithCredentials(creds)) + + // BigQuery 클라이언트 생성 + client, err := bigquery.NewClient(ctx, m.projectID, clientOptions...) if err != nil { - log.Warn("failed to create BigQuery client", zap.Error(err)) return nil, fmt.Errorf("failed to create BigQuery client: %v", err) } + return client, nil } +/* + func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { + creds, err := google.CredentialsFromJSON(ctx, []byte("JSON creds"), bigquery.Scope) + if err != nil { + log.Warn("failed to get crediential of client", zap.Error(err)) + return nil, fmt.Errorf("failed to get crediential of client: %v", err) + } + client, err := bigquery.NewClient(ctx, m.projectID, option.WithCredentials(creds)) + if err != nil { + log.Warn("failed to create BigQuery client", zap.Error(err)) + return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + } + return client, nil + } +*/ func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { retryFunc := func() error { q := m.client.Query(query) From 56c6d1241a494be275aa0ef477c06cbf5e548122 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 16:08:10 +0900 Subject: [PATCH 094/118] multi target database --- core/writer/bigquery_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index a4dea35c..f408676e 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -67,7 +67,7 @@ func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigque // TLS 인증서 검증을 비활성화하는 HTTP 클라이언트 설정 insecureTransport := &http.Transport{ TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, + InsecureSkipVerify: false, }, } httpClient := &http.Client{ From 2f78f6ff6f2be790eaae26459c3cfdc5b6f22088 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 21:15:51 +0900 Subject: [PATCH 095/118] multi target database --- core/writer/bigquery_handler.go | 46 +++------------------------------ 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index f408676e..4b85a09f 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -2,25 +2,21 @@ package writer import ( "context" - "crypto/tls" "fmt" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "net/http" "strings" "time" "cloud.google.com/go/bigquery" "github.com/cenkalti/backoff/v4" "github.com/cockroachdb/errors" - "go.uber.org/zap" - "google.golang.org/api/option" - "github.com/golang/protobuf/proto" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" + "go.uber.org/zap" ) type BigQueryDataHandler struct { @@ -64,52 +60,18 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi } func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - // TLS 인증서 검증을 비활성화하는 HTTP 클라이언트 설정 - insecureTransport := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: false, - }, - } - httpClient := &http.Client{ - Transport: insecureTransport, - } - - // 클라이언트 옵션 설정 - clientOptions := []option.ClientOption{ - option.WithHTTPClient(httpClient), - } - - // BigQuery 클라이언트 생성 - client, err := bigquery.NewClient(ctx, m.projectID, clientOptions...) + client, err := bigquery.NewClient(ctx, m.projectID) if err != nil { + log.Warn("failed to create BigQuery client", zap.Error(err)) return nil, fmt.Errorf("failed to create BigQuery client: %v", err) } - return client, nil } -/* - func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - creds, err := google.CredentialsFromJSON(ctx, []byte("JSON creds"), bigquery.Scope) - if err != nil { - log.Warn("failed to get crediential of client", zap.Error(err)) - return nil, fmt.Errorf("failed to get crediential of client: %v", err) - } - client, err := bigquery.NewClient(ctx, m.projectID, option.WithCredentials(creds)) - if err != nil { - log.Warn("failed to create BigQuery client", zap.Error(err)) - return nil, fmt.Errorf("failed to create BigQuery client: %v", err) - } - return client, nil - } -*/ func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { retryFunc := func() error { q := m.client.Query(query) - q.Parameters = make([]bigquery.QueryParameter, 0, len(params)) - for k, v := range params { - q.Parameters = append(q.Parameters, bigquery.QueryParameter{Name: k, Value: v}) - } + job, err := q.Run(ctx) if err != nil { log.Warn("failed to run query", zap.Error(err)) From 1cb18c9adceed0786cfd5d494fb99a549be88440 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 21:33:36 +0900 Subject: [PATCH 096/118] multi target database --- core/writer/bigquery_handler.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 4b85a09f..d41cd4d0 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -35,17 +35,11 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi connectTimeout: 5, retryOptions: backoff.NewExponentialBackOff(), } - handler.retryOptions.MaxElapsedTime = 2 * time.Minute // 설정된 최대 재시도 시간 + handler.retryOptions.MaxElapsedTime = 1 * time.Minute // 설정된 최대 재시도 시간 - for _, option := range options { - option.Apply(handler) - } if handler.projectID == "" { return nil, errors.New("empty BigQuery project ID") } - if handler.credentials == "" { - log.Warn("empty BigQuery credentials") - } var err error timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) @@ -60,6 +54,7 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi } func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { + log.Info("projectID", zap.String("projectID", m.projectID)) client, err := bigquery.NewClient(ctx, m.projectID) if err != nil { log.Warn("failed to create BigQuery client", zap.Error(err)) From 3ded3e20a67dd6759973ef00e8bfe829e4455c5b Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 22:08:09 +0900 Subject: [PATCH 097/118] multi target database --- core/writer/bigquery_handler.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index d41cd4d0..51ad7361 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -37,6 +37,9 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi } handler.retryOptions.MaxElapsedTime = 1 * time.Minute // 설정된 최대 재시도 시간 + for _, option := range options { + option.Apply(handler) + } if handler.projectID == "" { return nil, errors.New("empty BigQuery project ID") } @@ -54,7 +57,7 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi } func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - log.Info("projectID", zap.String("projectID", m.projectID)) + log.Info("createBigQueryClient", zap.String("projectID", m.projectID)) client, err := bigquery.NewClient(ctx, m.projectID) if err != nil { log.Warn("failed to create BigQuery client", zap.Error(err)) From 4e579b503a96e0877fa3749a3e1e6c7751044024 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 31 Jul 2024 22:31:05 +0900 Subject: [PATCH 098/118] multi target database --- core/writer/bigquery_handler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 51ad7361..29bbba90 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -23,7 +23,6 @@ type BigQueryDataHandler struct { api.DataHandler projectID string - credentials string connectTimeout int client *bigquery.Client retryOptions *backoff.ExponentialBackOff From 3071a524408321774a69fc411283160fce4cb6fe Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 1 Aug 2024 01:01:28 +0900 Subject: [PATCH 099/118] multi target database --- core/writer/bigquery_handler.go | 2 +- server/go.mod | 26 +++++++++++++------------- server/go.sum | 13 +++++++++++++ 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 29bbba90..254a0b88 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -40,7 +40,7 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi option.Apply(handler) } if handler.projectID == "" { - return nil, errors.New("empty BigQuery project ID") + return nil, fmt.Errorf("empty BigQuery project ID") } var err error diff --git a/server/go.mod b/server/go.mod index 51347513..e55d4a91 100644 --- a/server/go.mod +++ b/server/go.mod @@ -7,7 +7,7 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/goccy/go-json v0.10.2 github.com/golang/protobuf v1.5.4 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.4.0 github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240604033341-0f2b1f29bf47 github.com/milvus-io/milvus/pkg v0.0.2-0.20240618133959-0d20303e54ae @@ -19,16 +19,16 @@ require ( github.com/zilliztech/milvus-cdc/core v0.0.1 go.etcd.io/etcd/client/v3 v3.5.5 go.uber.org/zap v1.21.0 - google.golang.org/grpc v1.57.1 + google.golang.org/grpc v1.59.0 sigs.k8s.io/yaml v1.3.0 ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/bigquery v1.50.0 // indirect - cloud.google.com/go/compute v1.19.1 // indirect + cloud.google.com/go v0.110.8 // indirect + cloud.google.com/go/bigquery v1.57.1 // indirect + cloud.google.com/go/compute v1.23.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect + cloud.google.com/go/iam v1.1.3 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect github.com/AthenZ/athenz v1.10.39 // indirect @@ -70,8 +70,8 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -163,18 +163,18 @@ require ( golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.3.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.12.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.114.0 // indirect + google.golang.org/api v0.149.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect + google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/server/go.sum b/server/go.sum index 59c033fb..747a07fb 100644 --- a/server/go.sum +++ b/server/go.sum @@ -20,6 +20,7 @@ cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -28,8 +29,10 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.50.0 h1:RscMV6LbnAmhAzD893Lv9nXXy2WCaJmbxYPWDLbGqNQ= cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.57.1/go.mod h1:iYzC0tGVWt1jqSzBHqCr3lrRn0u13E8e+AqowBsDgug= cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP4c52gw= @@ -39,6 +42,7 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -367,12 +371,15 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -998,6 +1005,7 @@ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1012,6 +1020,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1208,6 +1217,7 @@ google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1264,8 +1274,10 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -1296,6 +1308,7 @@ google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5 google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQGivcXdTuY3Y9NBvr70X+y77woofSRluec= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From a05098d31e74096ce51fd953e698f0f52e3aa41c Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 1 Aug 2024 01:32:24 +0900 Subject: [PATCH 100/118] multi target database --- server/go.mod | 8 ++-- server/go.sum | 113 +++++++++++++++++++++++++++++++++----------------- 2 files changed, 79 insertions(+), 42 deletions(-) diff --git a/server/go.mod b/server/go.mod index e55d4a91..5c7b2bbf 100644 --- a/server/go.mod +++ b/server/go.mod @@ -25,7 +25,7 @@ require ( require ( cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/bigquery v1.57.1 // indirect + cloud.google.com/go/bigquery v1.56.0 // indirect cloud.google.com/go/compute v1.23.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.3 // indirect @@ -34,7 +34,7 @@ require ( github.com/AthenZ/athenz v1.10.39 // indirect github.com/DataDog/zstd v1.5.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/apache/arrow/go/v11 v11.0.0 // indirect + github.com/apache/arrow/go/v12 v12.0.0 // indirect github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 // indirect github.com/apache/thrift v0.16.0 // indirect github.com/ardielle/ardielle-go v1.5.2 // indirect @@ -53,7 +53,7 @@ require ( github.com/danieljoos/wincred v1.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -69,7 +69,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect diff --git a/server/go.sum b/server/go.sum index 747a07fb..d3b9f2a5 100644 --- a/server/go.sum +++ b/server/go.sum @@ -18,33 +18,36 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.50.0 h1:RscMV6LbnAmhAzD893Lv9nXXy2WCaJmbxYPWDLbGqNQ= -cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.56.0 h1:LHIc9E7Kw+ftFpQFKzZYBB88IAFz7qONawXXx0F3QBo= +cloud.google.com/go/bigquery v1.56.0/go.mod h1:KDcsploXTEY7XT3fDQzMUZlpQLHzE4itubHrnmhUrZA= +cloud.google.com/go/bigquery v1.57.1 h1:FiULdbbzUxWD0Y4ZGPSVCDLvqRSyCIO6zKV7E2nf5uA= cloud.google.com/go/bigquery v1.57.1/go.mod h1:iYzC0tGVWt1jqSzBHqCr3lrRn0u13E8e+AqowBsDgug= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/bigquery v1.62.0/go.mod h1:5ee+ZkF1x/ntgCsFQJAQTM3QkAZOecfCmvxhkJsWRSA= +cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= +cloud.google.com/go/compute v1.27.2/go.mod h1:YQuHkNEwP3bIz4LBYQqf4DIMfFtTDtnEgnwG0mJQQ9I= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP4c52gw= -cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/compute/metadata v0.4.0/go.mod h1:SIQh1Kkb4ZJ8zJ874fqVkslA29PRXuleyj6vOzlbK7M= +cloud.google.com/go/datacatalog v1.18.1 h1:xJp9mZrc2HPaoxIz3sP9pCmf/impifweQ/yGG9VBfio= +cloud.google.com/go/datacatalog v1.18.1/go.mod h1:TzAWaz+ON1tkNr4MOcak8EBHX7wIRX/gZKM+yTVsv+A= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= -cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/iam v1.1.10/go.mod h1:iEgMq62sg8zx446GCaijmA2Miwg5o3UbO+nI47WHJps= +cloud.google.com/go/longrunning v0.5.2 h1:u+oFqfEwwU7F9dIELigxbe0XVnBAo9wqMuQLA50CZ5k= +cloud.google.com/go/longrunning v0.5.2/go.mod h1:nqo6DQbNV2pXhGDbDMoN2bWz68MjZUzqv2YttZiveCs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -54,8 +57,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= @@ -91,8 +94,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow/go/v11 v11.0.0 h1:hqauxvFQxww+0mEU/2XHG6LT7eZternCZq+A5Yly2uM= -github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= @@ -187,8 +190,9 @@ github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4w github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= @@ -203,8 +207,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8= -github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -248,6 +252,7 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= @@ -281,8 +286,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -328,6 +333,7 @@ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -368,18 +374,20 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -483,6 +491,7 @@ github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ib github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -625,6 +634,7 @@ github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUM github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= @@ -741,6 +751,7 @@ github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -753,6 +764,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= @@ -801,6 +813,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= @@ -839,9 +852,11 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0 h1:g/BAN5o90Pr6D8xMRezjzGOHBpc15U+4oE53nZLiae4= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0/go.mod h1:+F41JBSkye7aYJELRvIMF0Z66reIwIOL0St75ZVwSJs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel v1.13.0 h1:1ZAKnNQKwBBxFtww/GwxNUyTf0AxkZzrukO8MeXqe4Y= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 h1:pa05sNT/P8OsIQ8mPZKTIyiBuzS/xDGLVx+DCt0y6Vs= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0/go.mod h1:rqbht/LlhVBgn5+k3M5QK96K5Xb0DvXpMJ5SFQpY6uw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= @@ -852,12 +867,15 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0 h1:Wz7UQ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0/go.mod h1:OhH1xvgA5jZW2M/S4PcvtDlFE1VULRRBsibBrKuJQGI= go.opentelemetry.io/otel/metric v0.35.0 h1:aPT5jk/w7F9zW51L7WgRqNKDElBdyRLGuBtI5MX34e8= go.opentelemetry.io/otel/metric v0.35.0/go.mod h1:qAcbhaTRFU6uG8QM7dDo7XvFsWcugziq/5YI065TokQ= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5JaM= go.opentelemetry.io/otel/sdk v1.13.0/go.mod h1:YLKPx5+6Vx/o1TCUYYs+bpymtkmazOMT6zoRrC7AQ7I= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= @@ -899,6 +917,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -911,6 +930,7 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI= golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -936,8 +956,10 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -988,8 +1010,10 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1003,9 +1027,9 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1017,10 +1041,11 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1098,13 +1123,17 @@ golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1114,8 +1143,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1183,14 +1214,17 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1215,9 +1249,9 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= +google.golang.org/api v0.188.0/go.mod h1:VR0d+2SIiWOYG3r/jdm7adPW9hI2aRv9ETOSCQ9Beag= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1226,6 +1260,7 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1272,14 +1307,15 @@ google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxH google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto v0.0.0-20240708141625-4ad9e859172b/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1306,9 +1342,9 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= -google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQGivcXdTuY3Y9NBvr70X+y77woofSRluec= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1327,6 +1363,7 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e76311e8aac9e0cd1c1db180fb266b8319043d27 Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 1 Aug 2024 02:03:23 +0900 Subject: [PATCH 101/118] multi target database --- core/writer/bigquery_handler.go | 61 +++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 254a0b88..585b80d1 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -2,27 +2,32 @@ package writer import ( "context" + "crypto/tls" "fmt" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" + "net/http" "strings" "time" "cloud.google.com/go/bigquery" "github.com/cenkalti/backoff/v4" "github.com/cockroachdb/errors" + "go.uber.org/zap" + "google.golang.org/api/option" + "github.com/golang/protobuf/proto" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" - "go.uber.org/zap" ) type BigQueryDataHandler struct { api.DataHandler projectID string + credentials string connectTimeout int client *bigquery.Client retryOptions *backoff.ExponentialBackOff @@ -34,13 +39,13 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi connectTimeout: 5, retryOptions: backoff.NewExponentialBackOff(), } - handler.retryOptions.MaxElapsedTime = 1 * time.Minute // 설정된 최대 재시도 시간 + handler.retryOptions.MaxElapsedTime = 2 * time.Minute // 설정된 최대 재시도 시간 for _, option := range options { option.Apply(handler) } if handler.projectID == "" { - return nil, fmt.Errorf("empty BigQuery project ID") + return nil, errors.New("empty BigQuery project ID") } var err error @@ -55,9 +60,50 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi return handler, nil } +/* func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - log.Info("createBigQueryClient", zap.String("projectID", m.projectID)) - client, err := bigquery.NewClient(ctx, m.projectID) + // TLS 인증서 검증을 비활성화하는 HTTP 클라이언트 설정 + insecureTransport := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: false, + }, + } + httpClient := &http.Client{ + Transport: insecureTransport, + } + + // 클라이언트 옵션 설정 + clientOptions := []option.ClientOption{ + option.WithHTTPClient(httpClient), + } + + // BigQuery 클라이언트 생성 + client, err := bigquery.NewClient(ctx, m.projectID, clientOptions...) + if err != nil { + return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + } + + return client, nil +} +*/ + +func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { + // TLS 인증서 검증을 비활성화하는 HTTP 클라이언트 설정 + insecureTransport := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + httpClient := &http.Client{ + Transport: insecureTransport, + } + + // 클라이언트 옵션 설정 + clientOptions := []option.ClientOption{ + option.WithHTTPClient(httpClient), + } + + client, err := bigquery.NewClient(ctx, m.projectID, clientOptions...) if err != nil { log.Warn("failed to create BigQuery client", zap.Error(err)) return nil, fmt.Errorf("failed to create BigQuery client: %v", err) @@ -68,7 +114,10 @@ func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigque func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { retryFunc := func() error { q := m.client.Query(query) - + q.Parameters = make([]bigquery.QueryParameter, 0, len(params)) + for k, v := range params { + q.Parameters = append(q.Parameters, bigquery.QueryParameter{Name: k, Value: v}) + } job, err := q.Run(ctx) if err != nil { log.Warn("failed to run query", zap.Error(err)) From ccc1ccd7c02a786a4877edc12aecbd81e012fea2 Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 1 Aug 2024 17:23:42 +0900 Subject: [PATCH 102/118] multi target database --- core/util/bigquery_client_resource.go | 3 ++- core/writer/bigquery_handler.go | 36 ++++++--------------------- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/core/util/bigquery_client_resource.go b/core/util/bigquery_client_resource.go index 0cc984ea..043d0767 100644 --- a/core/util/bigquery_client_resource.go +++ b/core/util/bigquery_client_resource.go @@ -60,8 +60,9 @@ func (m *ClientResourceManager) newBigQueryClient(ctx context.Context, projectId func (m *ClientResourceManager) GetBigQueryClient(ctx context.Context, projectId, database string) (*bigquery.Client, error) { if database == "" { - database = DefaultBigQueryDbName + return nil, errors.New("empty database name") } + ctxLog := log.Ctx(ctx).With(zap.String("database", database), zap.String("project_id", projectId)) res, err := m.manager.Get(BigQueryClientResourceTyp, getBigQueryClientResourceName(projectId, database), diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 585b80d1..8979f25e 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -2,32 +2,28 @@ package writer import ( "context" - "crypto/tls" "fmt" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "net/http" "strings" "time" "cloud.google.com/go/bigquery" "github.com/cenkalti/backoff/v4" "github.com/cockroachdb/errors" - "go.uber.org/zap" - "google.golang.org/api/option" - "github.com/golang/protobuf/proto" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" "github.com/zilliztech/milvus-cdc/core/log" + "go.uber.org/zap" ) type BigQueryDataHandler struct { api.DataHandler - projectID string - credentials string + projectID string + // credentials string connectTimeout int client *bigquery.Client retryOptions *backoff.ExponentialBackOff @@ -88,22 +84,7 @@ func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigque */ func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - // TLS 인증서 검증을 비활성화하는 HTTP 클라이언트 설정 - insecureTransport := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - httpClient := &http.Client{ - Transport: insecureTransport, - } - - // 클라이언트 옵션 설정 - clientOptions := []option.ClientOption{ - option.WithHTTPClient(httpClient), - } - - client, err := bigquery.NewClient(ctx, m.projectID, clientOptions...) + client, err := bigquery.NewClient(ctx, m.projectID) if err != nil { log.Warn("failed to create BigQuery client", zap.Error(err)) return nil, fmt.Errorf("failed to create BigQuery client: %v", err) @@ -114,10 +95,7 @@ func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigque func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { retryFunc := func() error { q := m.client.Query(query) - q.Parameters = make([]bigquery.QueryParameter, 0, len(params)) - for k, v := range params { - q.Parameters = append(q.Parameters, bigquery.QueryParameter{Name: k, Value: v}) - } + job, err := q.Run(ctx) if err != nil { log.Warn("failed to run query", zap.Error(err)) @@ -277,8 +255,8 @@ func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam func (m *BigQueryDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { query := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE `%s` = @value", param.Database, param.CollectionName, param.Column.Name()) - params := map[string]interface{}{"value": param.Column.FieldData()} - return m.bigqueryOp(ctx, query, params) + //params := map[string]interface{}{"value": param.Column.FieldData()} + return m.bigqueryOp(ctx, query, nil) } func (m *BigQueryDataHandler) CreateIndex(ctx context.Context, param *api.CreateIndexParam) error { From 3a99f11ffee85ae4dad9573e5135a56f8a5fc08a Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 1 Aug 2024 17:34:45 +0900 Subject: [PATCH 103/118] multi target database --- core/util/bigquery_client_resource.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/util/bigquery_client_resource.go b/core/util/bigquery_client_resource.go index 043d0767..aecd67e3 100644 --- a/core/util/bigquery_client_resource.go +++ b/core/util/bigquery_client_resource.go @@ -60,9 +60,9 @@ func (m *ClientResourceManager) newBigQueryClient(ctx context.Context, projectId func (m *ClientResourceManager) GetBigQueryClient(ctx context.Context, projectId, database string) (*bigquery.Client, error) { if database == "" { - return nil, errors.New("empty database name") + database = DefaultBigQueryDbName + log.Warn("empty database name") } - ctxLog := log.Ctx(ctx).With(zap.String("database", database), zap.String("project_id", projectId)) res, err := m.manager.Get(BigQueryClientResourceTyp, getBigQueryClientResourceName(projectId, database), From 65d8a95012d251812d64d03b4510370fd002d7d5 Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 1 Aug 2024 18:05:53 +0900 Subject: [PATCH 104/118] multi target database --- server/cdc_impl.go | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/server/cdc_impl.go b/server/cdc_impl.go index 292f6ebd..d97b5fa9 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -547,15 +547,30 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err ctx := context.TODO() timeoutCtx, cancelFunc := context.WithTimeout(ctx, time.Duration(milvusConnectParam.ConnectTimeout)*time.Second) - milvusClient, err := cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ - Address: milvusAddress, - Username: milvusConnectParam.Username, - Password: milvusConnectParam.Password, - EnableTLS: milvusConnectParam.EnableTLS, - DialConfig: milvusConnectParam.DialConfig, - ProjectId: milvusConnectParam.ProjectId, - TargetDBType: info.MilvusConnectParam.TargetDBType, - }) + var milvusClient api.TargetAPI + var err error + + log.Info("milvus connect param", zap.Any("milvus connect param", milvusConnectParam)) + if strings.ToLower(info.MilvusConnectParam.TargetDBType) == "milvus" { + milvusClient, err = cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ + Address: milvusAddress, + Username: milvusConnectParam.Username, + Password: milvusConnectParam.Password, + EnableTLS: milvusConnectParam.EnableTLS, + DialConfig: milvusConnectParam.DialConfig, + ProjectId: milvusConnectParam.ProjectId, + TargetDBType: info.MilvusConnectParam.TargetDBType, + }) + } else { + milvusClient, err = cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ + Address: milvusAddress, + Username: milvusConnectParam.Username, + Password: milvusConnectParam.Password, + EnableTLS: milvusConnectParam.EnableTLS, + ProjectId: milvusConnectParam.ProjectId, + TargetDBType: info.MilvusConnectParam.TargetDBType, + }) + } cancelFunc() if err != nil { From 630a54163b32e86b9c7f142bad2b7c8fa8240243 Mon Sep 17 00:00:00 2001 From: ianlee Date: Thu, 1 Aug 2024 19:56:51 +0900 Subject: [PATCH 105/118] multi target database --- core/writer/bigquery_handler.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 8979f25e..5408b7d9 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -250,7 +250,23 @@ func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam param.Database, param.CollectionName, join(columns, ","), values) log.Info("INSERT", zap.String("query", query)) - return m.bigqueryOp(ctx, query, nil) + q := m.client.Query(query) + + job, err := q.Run(ctx) + if err != nil { + log.Warn("failed to run query", zap.Error(err)) + return err + } + status, err := job.Wait(ctx) + if err != nil { + return err + } + if err := status.Err(); err != nil { + return err + } + return nil + + //return m.bigqueryOp(ctx, query, nil) } func (m *BigQueryDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { From 7b7f72925e8f947907b2544b87d49142dd521ca6 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 2 Aug 2024 12:04:03 +0900 Subject: [PATCH 106/118] multi target database --- core/writer/bigquery_handler.go | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 5408b7d9..67359ed9 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -48,7 +48,7 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) defer cancel() - handler.client, err = handler.createBigQueryClient(timeoutContext) + err = handler.createBigQueryClient(timeoutContext) if err != nil { return nil, err } @@ -83,13 +83,15 @@ func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigque } */ -func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - client, err := bigquery.NewClient(ctx, m.projectID) +func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (err error) { + m.client, err = bigquery.NewClient(ctx, m.projectID) if err != nil { log.Warn("failed to create BigQuery client", zap.Error(err)) - return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + return fmt.Errorf("failed to create BigQuery client: %v", err) } - return client, nil + + log.Info("BigQuery client created", zap.String("projectID", m.projectID)) + return nil } func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { @@ -250,23 +252,7 @@ func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam param.Database, param.CollectionName, join(columns, ","), values) log.Info("INSERT", zap.String("query", query)) - q := m.client.Query(query) - - job, err := q.Run(ctx) - if err != nil { - log.Warn("failed to run query", zap.Error(err)) - return err - } - status, err := job.Wait(ctx) - if err != nil { - return err - } - if err := status.Err(); err != nil { - return err - } - return nil - - //return m.bigqueryOp(ctx, query, nil) + return m.bigqueryOp(ctx, query, nil) } func (m *BigQueryDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { @@ -413,6 +399,7 @@ func (m *BigQueryDataHandler) unmarshalTsMsg(ctx context.Context, msgType common } func (m *BigQueryDataHandler) ReplicateMessage(ctx context.Context, param *api.ReplicateMessageParam) error { + param.Database = strings.Split(param.ChannelName, "-")[0] for i, msgBytes := range param.MsgsBytes { header := &commonpb.MsgHeader{} err := proto.Unmarshal(msgBytes, header) @@ -426,7 +413,6 @@ func (m *BigQueryDataHandler) ReplicateMessage(ctx context.Context, param *api.R return err } - param.Database = strings.Split(param.ChannelName, "-")[0] err = m.unmarshalTsMsg(ctx, header.GetBase().GetMsgType(), param.Database, msgBytes) if err != nil { log.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err)) From 098d116ec3356db908adb612cb0bd5521b316521 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 2 Aug 2024 12:14:57 +0900 Subject: [PATCH 107/118] multi target database --- core/writer/bigquery_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 67359ed9..a3e8f2d9 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -96,6 +96,7 @@ func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (err err func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { retryFunc := func() error { + query = "select count(*) from `cdctest`.`replica_collection`" q := m.client.Query(query) job, err := q.Run(ctx) From 5a0276c3108fc47c57203c3413f653c203f8816d Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 2 Aug 2024 15:24:12 +0900 Subject: [PATCH 108/118] multi target database --- core/util/common_client_resource.go | 10 +++ core/writer/bigquery_handler.go | 102 ++++++++++++++++++++++------ core/writer/mysql_handler.go | 10 +-- 3 files changed, 95 insertions(+), 27 deletions(-) diff --git a/core/util/common_client_resource.go b/core/util/common_client_resource.go index 1db9e0ab..c16d818f 100644 --- a/core/util/common_client_resource.go +++ b/core/util/common_client_resource.go @@ -28,3 +28,13 @@ func GetMilvusClientManager() *ClientResourceManager { func GetAPIKey(username, password string) string { return fmt.Sprintf("%s:%s", username, password) } + +func GetBigQueryClientManager() *ClientResourceManager { + clientManagerOnce.Do(func() { + manager := resource.NewManager(0, 0, nil) + clientManager = &ClientResourceManager{ + manager: manager, + } + }) + return clientManager +} diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index a3e8f2d9..13b1a937 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -6,11 +6,12 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" + "github.com/milvus-io/milvus/pkg/util/retry" + "github.com/zilliztech/milvus-cdc/core/util" "strings" "time" "cloud.google.com/go/bigquery" - "github.com/cenkalti/backoff/v4" "github.com/cockroachdb/errors" "github.com/golang/protobuf/proto" "github.com/zilliztech/milvus-cdc/core/api" @@ -26,16 +27,16 @@ type BigQueryDataHandler struct { // credentials string connectTimeout int client *bigquery.Client - retryOptions *backoff.ExponentialBackOff + retryOptions []retry.Option } // NewBigQueryDataHandler options must include ProjectIDOption and CredentialsOption func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*BigQueryDataHandler, error) { handler := &BigQueryDataHandler{ connectTimeout: 5, - retryOptions: backoff.NewExponentialBackOff(), } - handler.retryOptions.MaxElapsedTime = 2 * time.Minute // 설정된 최대 재시도 시간 + + handler.retryOptions = util.GetRetryOptions(config.GetCommonConfig().Retry) for _, option := range options { option.Apply(handler) @@ -94,33 +95,78 @@ func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (err err return nil } -func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { - retryFunc := func() error { - query = "select count(*) from `cdctest`.`replica_collection`" - q := m.client.Query(query) +func executeQuery(ctx context.Context, bigqueryClient *bigquery.Client, query string) error { + q := bigqueryClient.Query(query) - job, err := q.Run(ctx) - if err != nil { - log.Warn("failed to run query", zap.Error(err)) + job, err := q.Run(ctx) + if err != nil { + log.Warn("failed to run query", zap.Error(err)) + return err + } + status, err := job.Wait(ctx) + if err != nil { + return err + } + if err := status.Err(); err != nil { + return err + } + return nil + +} +func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, f func(bigqueryClient *bigquery.Client) error) error { + retryBigQueryFunc := func(c *bigquery.Client) error { + // TODO Retryable and non-retryable errors should be distinguished + var err error + retryErr := retry.Do(ctx, func() error { + err = f(c) return err - } - status, err := job.Wait(ctx) - if err != nil { + }, m.retryOptions...) + if retryErr != nil && err != nil { return err } - if err := status.Err(); err != nil { - return err + if retryErr != nil { + return retryErr } return nil } - err := backoff.Retry(retryFunc, backoff.WithContext(m.retryOptions, ctx)) + bigqueryClient, err := util.GetBigQueryClientManager().GetBigQueryClient(ctx, m.projectID, "") if err != nil { - log.Warn("retry operation failed", zap.Error(err)) + log.Warn("fail to get bigquery client", zap.Error(err)) + return err } - return err + + return retryBigQueryFunc(bigqueryClient) } +/* + func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { + retryFunc := func() error { + + q := m.client.Query(query) + + job, err := q.Run(ctx) + if err != nil { + log.Warn("failed to run query", zap.Error(err)) + return err + } + status, err := job.Wait(ctx) + if err != nil { + return err + } + if err := status.Err(); err != nil { + return err + } + return nil + } + + err := backoff.Retry(retryFunc, backoff.WithContext(m.retryOptions, ctx)) + if err != nil { + log.Warn("retry operation failed", zap.Error(err)) + } + return err + } +*/ func (m *BigQueryDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { schema := bigquery.Schema{} for _, field := range param.Schema.Fields { @@ -253,19 +299,31 @@ func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam param.Database, param.CollectionName, join(columns, ","), values) log.Info("INSERT", zap.String("query", query)) - return m.bigqueryOp(ctx, query, nil) + log.Info("insert request", zap.String("collection", param.CollectionName), zap.Any("columns", param.Columns)) + return m.bigqueryOp(ctx, func(bigqueryClient *bigquery.Client) error { + err := executeQuery(ctx, bigqueryClient, query) + return err + }) + + //return m.bigqueryOp(ctx, executeQuery(ctx context.Context, bigqueryClient *bigquery.Client, query string)) } func (m *BigQueryDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { query := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE `%s` = @value", param.Database, param.CollectionName, param.Column.Name()) //params := map[string]interface{}{"value": param.Column.FieldData()} - return m.bigqueryOp(ctx, query, nil) + return m.bigqueryOp(ctx, func(bigqueryClient *bigquery.Client) error { + err := executeQuery(ctx, bigqueryClient, query) + return err + }) } func (m *BigQueryDataHandler) CreateIndex(ctx context.Context, param *api.CreateIndexParam) error { // BigQuery는 Vector Search Index만을 지원합니다. query := fmt.Sprintf("CREATE OR REPLACE VECTOR INDEX `%s`.%s ON `%s`(embedding_v1) OPTIONS(distance_type='L2',index_type_type='IVF')", param.Database, param.IndexName, param.CollectionName) - return m.bigqueryOp(ctx, query, nil) + return m.bigqueryOp(ctx, func(bigqueryClient *bigquery.Client) error { + err := executeQuery(ctx, bigqueryClient, query) + return err + }) } func (m *BigQueryDataHandler) DropIndex(ctx context.Context, param *api.DropIndexParam) error { diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 1b1a34f6..0e294348 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -50,7 +50,7 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat var err error - handler.db, err = handler.createDBConnection(handler.connectTimeout) + err = handler.createDBConnection(handler.connectTimeout) if err != nil { log.Error("failed to create mysql connection", zap.Error(err)) return nil, err @@ -59,7 +59,7 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat return handler, nil } -func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, error) { +func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (err error) { cfg := mysql.Config{ Addr: m.address, User: m.username, @@ -69,13 +69,13 @@ func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (*sql.DB, e Timeout: time.Duration(connectionTimeout) * time.Second, } - db, err := sql.Open("mysql", cfg.FormatDSN()) + m.db, err = sql.Open("mysql", cfg.FormatDSN()) if err != nil { log.Info("failed to connect mysql database", zap.Any("info", cfg.FormatDSN())) - return nil, fmt.Errorf("failed to connect mysql database : %v", err) + return fmt.Errorf("failed to connect mysql database : %v", err) } - return db, nil + return nil } func (m *MySQLDataHandler) mysqlOp(ctx context.Context, query string, args ...interface{}) error { From 2cd670257c0e92b33f853f787a6cdc39a413f719 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 2 Aug 2024 15:41:45 +0900 Subject: [PATCH 109/118] multi target database --- core/writer/bigquery_handler.go | 44 ++++----------------------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 13b1a937..38dee5e1 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -49,52 +49,18 @@ func NewBigQueryDataHandler(options ...config.Option[*BigQueryDataHandler]) (*Bi timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) defer cancel() - err = handler.createBigQueryClient(timeoutContext) + err = handler.bigqueryOp(timeoutContext, func(bigquery *bigquery.Client) error { + return nil + }) if err != nil { + log.Warn("fail to new the milvus client", zap.Error(err)) return nil, err } + handler.retryOptions = util.GetRetryOptions(config.GetCommonConfig().Retry) return handler, nil } -/* -func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - // TLS 인증서 검증을 비활성화하는 HTTP 클라이언트 설정 - insecureTransport := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: false, - }, - } - httpClient := &http.Client{ - Transport: insecureTransport, - } - - // 클라이언트 옵션 설정 - clientOptions := []option.ClientOption{ - option.WithHTTPClient(httpClient), - } - - // BigQuery 클라이언트 생성 - client, err := bigquery.NewClient(ctx, m.projectID, clientOptions...) - if err != nil { - return nil, fmt.Errorf("failed to create BigQuery client: %v", err) - } - - return client, nil -} -*/ - -func (m *BigQueryDataHandler) createBigQueryClient(ctx context.Context) (err error) { - m.client, err = bigquery.NewClient(ctx, m.projectID) - if err != nil { - log.Warn("failed to create BigQuery client", zap.Error(err)) - return fmt.Errorf("failed to create BigQuery client: %v", err) - } - - log.Info("BigQuery client created", zap.String("projectID", m.projectID)) - return nil -} - func executeQuery(ctx context.Context, bigqueryClient *bigquery.Client, query string) error { q := bigqueryClient.Query(query) From 9a2acb539902a9936556c79870f6746078b829b7 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 2 Aug 2024 16:43:45 +0900 Subject: [PATCH 110/118] multi target database --- core/reader/target_client.go | 2 +- core/util/common_client_resource.go | 10 ++++++++++ core/util/mysql_client_resource.go | 19 ++++++++++++------- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 2ce89dfb..8d144748 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -44,7 +44,7 @@ type TargetClient struct { milvusClient client.Client config TargetConfig mysqlClient driver.Connector - bigqueryClient bigquery.Client + bigqueryClient *bigquery.Client } type TargetConfig struct { diff --git a/core/util/common_client_resource.go b/core/util/common_client_resource.go index c16d818f..a2d0cc5c 100644 --- a/core/util/common_client_resource.go +++ b/core/util/common_client_resource.go @@ -38,3 +38,13 @@ func GetBigQueryClientManager() *ClientResourceManager { }) return clientManager } + +func GetMySqlClientManager() *ClientResourceManager { + clientManagerOnce.Do(func() { + manager := resource.NewManager(0, 0, nil) + clientManager = &ClientResourceManager{ + manager: manager, + } + }) + return clientManager +} diff --git a/core/util/mysql_client_resource.go b/core/util/mysql_client_resource.go index 276763cb..5844fcf6 100644 --- a/core/util/mysql_client_resource.go +++ b/core/util/mysql_client_resource.go @@ -39,22 +39,27 @@ const ( MySQLClientResourceTyp = "mysql_client" MySQLClientExpireTime = 30 * time.Second DefaultMySQLDbName = "information_schema" + connectionTimeout = 1 ) func (m *ClientResourceManager) newMySQLClient(ctx context.Context, address, apiKey, database string, enableTLS bool, dialConfig DialConfig) resource.NewResourceFunc { return func() (resource.Resource, error) { - c, err := mysql.NewConnector(&mysql.Config{ - Addr: address, - User: strings.Split(apiKey, ":")[0], - Passwd: strings.Split(apiKey, ":")[1], - DBName: database, - }) + + cfg := mysql.Config{ + Addr: address, + User: strings.Split(apiKey, ":")[0], + Passwd: strings.Split(apiKey, ":")[1], + Net: "tcp", + AllowNativePasswords: true, + Timeout: time.Duration(connectionTimeout) * time.Second, + } + + db, err := sql.Open("mysql", cfg.FormatDSN()) if err != nil { log.Warn("fail to new the mysql client", zap.String("database", database), zap.String("address", address), zap.Error(err)) return nil, err } - db := sql.OpenDB(c) res := resource.NewSimpleResource(db, MySQLClientResourceTyp, fmt.Sprintf("%s:%s", address, database), MySQLClientExpireTime, func() { _ = db.Close() }) From ad2cef12c9f6a23a224d45024273da6991533977 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 2 Aug 2024 17:59:58 +0900 Subject: [PATCH 111/118] multi target database --- core/util/bigquery_client_resource.go | 10 ++++------ server/cdc_impl.go | 9 ++++----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/core/util/bigquery_client_resource.go b/core/util/bigquery_client_resource.go index aecd67e3..b67d9cf8 100644 --- a/core/util/bigquery_client_resource.go +++ b/core/util/bigquery_client_resource.go @@ -42,9 +42,7 @@ const ( func (m *ClientResourceManager) newBigQueryClient(ctx context.Context, projectId string) resource.NewResourceFunc { return func() (resource.Resource, error) { - c, err := bigquery.NewClient(ctx, - projectId, - ) + c, err := bigquery.NewClient(ctx, projectId) if err != nil { log.Warn("fail to new the bigquery client", zap.String("project_id", projectId), zap.Error(err)) return nil, err @@ -65,7 +63,7 @@ func (m *ClientResourceManager) GetBigQueryClient(ctx context.Context, projectId } ctxLog := log.Ctx(ctx).With(zap.String("database", database), zap.String("project_id", projectId)) res, err := m.manager.Get(BigQueryClientResourceTyp, - getBigQueryClientResourceName(projectId, database), + getBigQueryClientResourceName(projectId), m.newBigQueryClient(ctx, projectId)) if err != nil { ctxLog.Error("fail to get bigquery client", zap.Error(err)) @@ -78,6 +76,6 @@ func (m *ClientResourceManager) GetBigQueryClient(ctx context.Context, projectId return nil, errors.New("invalid resource object") } -func getBigQueryClientResourceName(projectId, database string) string { - return fmt.Sprintf("%s:%s", projectId, database) +func getBigQueryClientResourceName(projectId string) string { + return fmt.Sprintf("%s", projectId) } diff --git a/server/cdc_impl.go b/server/cdc_impl.go index d97b5fa9..f26b70ae 100644 --- a/server/cdc_impl.go +++ b/server/cdc_impl.go @@ -551,7 +551,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err var err error log.Info("milvus connect param", zap.Any("milvus connect param", milvusConnectParam)) - if strings.ToLower(info.MilvusConnectParam.TargetDBType) == "milvus" { + if strings.ToLower(milvusConnectParam.TargetDBType) == "milvus" { milvusClient, err = cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ Address: milvusAddress, Username: milvusConnectParam.Username, @@ -559,7 +559,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err EnableTLS: milvusConnectParam.EnableTLS, DialConfig: milvusConnectParam.DialConfig, ProjectId: milvusConnectParam.ProjectId, - TargetDBType: info.MilvusConnectParam.TargetDBType, + TargetDBType: milvusConnectParam.TargetDBType, }) } else { milvusClient, err = cdcreader.NewTarget(timeoutCtx, cdcreader.TargetConfig{ @@ -568,7 +568,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err Password: milvusConnectParam.Password, EnableTLS: milvusConnectParam.EnableTLS, ProjectId: milvusConnectParam.ProjectId, - TargetDBType: info.MilvusConnectParam.TargetDBType, + TargetDBType: milvusConnectParam.TargetDBType, }) } cancelFunc() @@ -579,8 +579,7 @@ func (e *MetaCDC) newReplicateEntity(info *meta.TaskInfo) (*ReplicateEntity, err } taskLog.Info("taskInfo", zap.Any("taskInfo", info)) - taskLog.Info("target db type", zap.Any("target db type", info.MilvusConnectParam.TargetDBType)) - taskLog.Info("milvusClient", zap.Any("milvusClient", milvusClient)) + taskLog.Info("target db type", zap.Any("target db type", milvusConnectParam.TargetDBType)) sourceConfig := e.config.SourceConfig etcdServerConfig := GetEtcdServerConfigFromSourceConfig(sourceConfig) From 74f2034df0ac9580ad5cca8b05927967d4644b82 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 2 Aug 2024 18:18:31 +0900 Subject: [PATCH 112/118] multi target database --- core/reader/target_client.go | 10 +++++----- core/util/bigquery_client_resource.go | 8 ++------ core/writer/bigquery_handler.go | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 8d144748..9a67ac31 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -76,7 +76,7 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) return nil, err } } else if strings.ToLower(targetClient.config.TargetDBType) == "bigquery" { - _, err := targetClient.GetBigQuery(ctx, "") + _, err := targetClient.GetBigQuery(ctx) if err != nil { log.Warn("fail to new target client", zap.String("address", config.ProjectId), zap.Error(err)) return nil, err @@ -104,15 +104,15 @@ func (t *TargetClient) GetMySQL(ctx context.Context, databaseName string) (*sql. if apiKey == "" { apiKey = util.GetAPIKey(t.config.Username, t.config.Password) } - mysqlClient, err := util.GetMilvusClientManager().GetMySQLClient(ctx, t.config.Address, apiKey, databaseName, t.config.EnableTLS, t.config.DialConfig) + mysqlClient, err := util.GetMySqlClientManager().GetMySQLClient(ctx, t.config.Address, apiKey, databaseName, t.config.EnableTLS, t.config.DialConfig) if err != nil { return nil, err } return mysqlClient, nil } -func (t *TargetClient) GetBigQuery(ctx context.Context, databaseName string) (*bigquery.Client, error) { - bigqueryClient, err := util.GetMilvusClientManager().GetBigQueryClient(ctx, t.config.ProjectId, databaseName) +func (t *TargetClient) GetBigQuery(ctx context.Context) (*bigquery.Client, error) { + bigqueryClient, err := util.GetBigQueryClientManager().GetBigQueryClient(ctx, t.config.ProjectId) if err != nil { return nil, err } @@ -280,7 +280,7 @@ func (t *TargetClient) GetDatabaseName(ctx context.Context, collectionName, data } else if strings.ToLower(t.config.TargetDBType) == "bigquery" { dbLog := log.With(zap.String("table", collectionName), zap.String("database", databaseName)) - dbBigQuery, err := t.GetBigQuery(ctx, "") + dbBigQuery, err := t.GetBigQuery(ctx) if err != nil { dbLog.Warn("fail to mysql list tables", zap.String("connect_db", databaseName), zap.Error(err)) return "", err diff --git a/core/util/bigquery_client_resource.go b/core/util/bigquery_client_resource.go index b67d9cf8..5fb9e31b 100644 --- a/core/util/bigquery_client_resource.go +++ b/core/util/bigquery_client_resource.go @@ -56,12 +56,8 @@ func (m *ClientResourceManager) newBigQueryClient(ctx context.Context, projectId } } -func (m *ClientResourceManager) GetBigQueryClient(ctx context.Context, projectId, database string) (*bigquery.Client, error) { - if database == "" { - database = DefaultBigQueryDbName - log.Warn("empty database name") - } - ctxLog := log.Ctx(ctx).With(zap.String("database", database), zap.String("project_id", projectId)) +func (m *ClientResourceManager) GetBigQueryClient(ctx context.Context, projectId string) (*bigquery.Client, error) { + ctxLog := log.Ctx(ctx).With(zap.String("project_id", projectId)) res, err := m.manager.Get(BigQueryClientResourceTyp, getBigQueryClientResourceName(projectId), m.newBigQueryClient(ctx, projectId)) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 38dee5e1..1311efb4 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -96,7 +96,7 @@ func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, f func(bigqueryCli return nil } - bigqueryClient, err := util.GetBigQueryClientManager().GetBigQueryClient(ctx, m.projectID, "") + bigqueryClient, err := util.GetBigQueryClientManager().GetBigQueryClient(ctx, m.projectID) if err != nil { log.Warn("fail to get bigquery client", zap.Error(err)) return err From 7d84e05d3d3b658a2ff28b395526b676e03fb464 Mon Sep 17 00:00:00 2001 From: ianlee Date: Fri, 2 Aug 2024 20:22:01 +0900 Subject: [PATCH 113/118] multi target database --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index e66f1fb5..dc056d1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +RUN apt-get update && apt-get install -y ca-certificates + FROM golang:1.21 AS builder ENV CGO_ENABLED=1 ARG GIT_COMMIT_ARG From c8d895e8469383d9dbe7b83054e0f1d88a57800a Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 5 Aug 2024 14:53:41 +0900 Subject: [PATCH 114/118] multi target database --- Dockerfile | 3 +-- core/writer/bigquery_handler.go | 35 +++++++++++++++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc056d1a..860bd69a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,3 @@ -RUN apt-get update && apt-get install -y ca-certificates - FROM golang:1.21 AS builder ENV CGO_ENABLED=1 ARG GIT_COMMIT_ARG @@ -10,6 +8,7 @@ RUN cd server && make build && mv ../bin/cdc /app/milvus-cdc FROM debian:bookworm WORKDIR /app +RUN apt-get update && apt-get install -y ca-certificates COPY --from=builder /app/milvus-cdc ./ COPY --from=builder /app/server/configs ./configs EXPOSE 8444 diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index 1311efb4..b0939333 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -153,7 +153,7 @@ func (m *BigQueryDataHandler) DropCollection(ctx context.Context, param *api.Dro return tableRef.Delete(ctx) } -func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { +func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam) (err error) { columns := []string{} rowValues := [][]interface{}{} @@ -232,7 +232,8 @@ func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam } } - var value, values string + var value string + var values []string for rowCnt := 0; rowCnt < len(rowValues[0]); rowCnt++ { for colNo, _ := range columns { @@ -253,25 +254,25 @@ func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam } value = fmt.Sprintf("%s)", value) - if rowCnt == 0 { - values = fmt.Sprintf("%s", value) - } else { - values = fmt.Sprintf("%s,%s", values, value) - } + values = append(values, value) } - //string_to_vector('[1,2,3]') - query := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s", - param.Database, param.CollectionName, join(columns, ","), values) - log.Info("INSERT", zap.String("query", query)) + for _, v := range values { + query := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s", + param.Database, param.CollectionName, join(columns, ","), v) + log.Info("INSERT", zap.String("query", query)) - log.Info("insert request", zap.String("collection", param.CollectionName), zap.Any("columns", param.Columns)) - return m.bigqueryOp(ctx, func(bigqueryClient *bigquery.Client) error { - err := executeQuery(ctx, bigqueryClient, query) - return err - }) + err = m.bigqueryOp(ctx, func(bigqueryClient *bigquery.Client) error { + inerr := executeQuery(ctx, bigqueryClient, query) + return inerr + }) - //return m.bigqueryOp(ctx, executeQuery(ctx context.Context, bigqueryClient *bigquery.Client, query string)) + if err != nil { + return err + } + } + + return nil } func (m *BigQueryDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { From 58b92909f8857307f7fbbd48994de331f820bb8e Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 5 Aug 2024 15:19:50 +0900 Subject: [PATCH 115/118] multi target database --- core/writer/bigquery_handler.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index b0939333..f0191f72 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -237,19 +237,10 @@ func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam for rowCnt := 0; rowCnt < len(rowValues[0]); rowCnt++ { for colNo, _ := range columns { - switch rowValues[colNo][rowCnt].(type) { - case string: - if colNo == 0 { - value = fmt.Sprintf("(%s", fmt.Sprintf("%s", rowValues[colNo][rowCnt])) - } else { - value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", rowValues[colNo][rowCnt])) - } - default: - if colNo == 0 { - value = fmt.Sprintf("(%s", fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) - } else { - value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) - } + if colNo == 0 { + value = fmt.Sprintf("(%s", fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + } else { + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) } } value = fmt.Sprintf("%s)", value) From 8ed3e7a85ae1267fe8ab11d95cbd662e792a30ad Mon Sep 17 00:00:00 2001 From: ianlee Date: Mon, 5 Aug 2024 15:30:02 +0900 Subject: [PATCH 116/118] multi target database --- core/writer/bigquery_handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index f0191f72..cda140e9 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -238,9 +238,9 @@ func (m *BigQueryDataHandler) Insert(ctx context.Context, param *api.InsertParam for rowCnt := 0; rowCnt < len(rowValues[0]); rowCnt++ { for colNo, _ := range columns { if colNo == 0 { - value = fmt.Sprintf("(%s", fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + value = fmt.Sprintf("(%s", fmt.Sprintf("%v", rowValues[colNo][rowCnt])) } else { - value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) + value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%v", rowValues[colNo][rowCnt])) } } value = fmt.Sprintf("%s)", value) From 51b00a351f77a3c58ab8a595d023d8c1d9fd9582 Mon Sep 17 00:00:00 2001 From: ianlee Date: Wed, 7 Aug 2024 16:34:46 +0900 Subject: [PATCH 117/118] multi target database --- core/go.mod | 85 +++-- core/go.sum | 273 ++++++++-------- core/reader/target_client.go | 39 ++- core/util/elasticsearch_client_resource.go | 104 ++++++ core/util/mysql_client_resource.go | 19 +- core/writer/bigquery_handler.go | 28 -- core/writer/es_handler.go | 248 +++++++++------ core/writer/mysql_handler.go | 125 +++++--- server/go.mod | 113 ++++--- server/go.sum | 350 +++++++-------------- server/tool/msg_count/main.go | 5 +- 11 files changed, 733 insertions(+), 656 deletions(-) create mode 100644 core/util/elasticsearch_client_resource.go diff --git a/core/go.mod b/core/go.mod index d4c681aa..cec93601 100644 --- a/core/go.mod +++ b/core/go.mod @@ -3,7 +3,10 @@ module github.com/zilliztech/milvus-cdc/core go 1.21 require ( + cloud.google.com/go/bigquery v1.53.0 github.com/cockroachdb/errors v1.9.1 + github.com/elastic/go-elasticsearch/v8 v8.14.0 + github.com/go-sql-driver/mysql v1.5.0 github.com/golang/protobuf v1.5.4 github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240604033341-0f2b1f29bf47 @@ -12,23 +15,31 @@ require ( github.com/samber/lo v1.27.0 github.com/sasha-s/go-deadlock v0.3.1 github.com/stretchr/testify v1.8.4 - go.etcd.io/etcd/client/v3 v3.5.5 + go.etcd.io/etcd/client/v3 v3.5.15 go.uber.org/zap v1.21.0 golang.org/x/time v0.5.0 - google.golang.org/grpc v1.57.1 + google.golang.org/api v0.126.0 + google.golang.org/grpc v1.59.0 ) require ( + cloud.google.com/go v0.110.7 // indirect + cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect github.com/AthenZ/athenz v1.10.39 // indirect github.com/DataDog/zstd v1.5.0 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect + github.com/apache/arrow/go/v12 v12.0.0 // indirect github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 // indirect + github.com/apache/thrift v0.16.0 // indirect github.com/ardielle/ardielle-go v1.5.2 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cilium/ebpf v0.11.0 // indirect github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect @@ -40,30 +51,41 @@ require ( github.com/danieljoos/wincred v1.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/getsentry/sentry-go v0.12.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/goccy/go-json v0.9.11 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect + github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/s2a-go v0.1.4 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/linkedin/goavro/v2 v2.11.1 // indirect @@ -71,6 +93,8 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect + github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/minio/highwayhash v1.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -86,6 +110,7 @@ require ( github.com/pelletier/go-toml v1.9.3 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pierrec/lz4 v2.5.2+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -94,7 +119,7 @@ require ( github.com/prometheus/procfs v0.9.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/shirou/gopsutil/v3 v3.22.9 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.6.0 // indirect @@ -116,36 +141,40 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - go.etcd.io/etcd/api/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/v2 v2.305.5 // indirect - go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect - go.etcd.io/etcd/raft/v3 v3.5.5 // indirect - go.etcd.io/etcd/server/v3 v3.5.5 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0 // indirect - go.opentelemetry.io/otel v1.13.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0 // indirect - go.opentelemetry.io/otel/metric v0.35.0 // indirect - go.opentelemetry.io/otel/sdk v1.13.0 // indirect - go.opentelemetry.io/otel/trace v1.13.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/etcd/api/v3 v3.5.15 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect + go.etcd.io/etcd/client/v2 v2.305.15 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.15 // indirect + go.etcd.io/etcd/raft/v3 v3.5.15 // indirect + go.etcd.io/etcd/server/v3 v3.5.15 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/sdk v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect + golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.19.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.12.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/core/go.sum b/core/go.sum index bae5cb04..6adc5476 100644 --- a/core/go.sum +++ b/core/go.sum @@ -18,20 +18,29 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/bigquery v1.53.0 h1:K3wLbjbnSlxhuG5q4pntHv5AEbQM1QqHKGYgwFIqOTg= +cloud.google.com/go/bigquery v1.53.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/datacatalog v1.16.0 h1:qVeQcw1Cz93/cGu2E7TYUPh8Lz5dn5Ws2siIuQ17Vng= +cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -41,6 +50,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= @@ -57,8 +68,9 @@ github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/SimFG/milvus/pkg v0.0.0-20240628091152-f303bc30eaaa h1:YIuRENItb5LGPQy500gX977/TkUEtMUzfD/ujK4qJTg= github.com/SimFG/milvus/pkg v0.0.0-20240628091152-f303bc30eaaa/go.mod h1:XT4MPwUYb+UcvbBINVmqqcbePR3+KCEeBqYm6p2f/0A= @@ -71,7 +83,13 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= @@ -88,18 +106,13 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -113,20 +126,16 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= @@ -136,52 +145,49 @@ github.com/confluentinc/confluent-kafka-go v1.9.1 h1:L3aW6KvTyrq/+BOMnDm9xJylhAE github.com/confluentinc/confluent-kafka-go v1.9.1/go.mod h1:ptXNqsuDfYbAE/LBW6pnwWZElUoWxHoV8E43DCrliyo= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= +github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow= +github.com/elastic/go-elasticsearch/v8 v8.14.0/go.mod h1:WRvnlGkSuZyp83M2U8El/LGXpCjYLrvlkSgkAH4O5I4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8= -github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -201,7 +207,6 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -223,19 +228,22 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= @@ -244,22 +252,21 @@ github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q8 github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -295,9 +302,10 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -318,9 +326,12 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -335,11 +346,18 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -347,16 +365,14 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hamba/avro v1.5.6/go.mod h1:3vNT0RLXXpFm2Tb/5KC71ZRJlOroggq1Rcitb6k4Fr8= @@ -406,7 +422,6 @@ github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7 github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -429,16 +444,18 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -469,7 +486,6 @@ github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= @@ -501,6 +517,10 @@ github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240604033341-0f2b1f29bf47 h1:4T github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240604033341-0f2b1f29bf47/go.mod h1:6ckCQ8h5iFncZcaIpGE8T8sqliwaw2Eu9UzHaCURMA4= github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A= github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -544,7 +564,6 @@ github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -575,6 +594,8 @@ github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCr github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= @@ -592,7 +613,6 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:Om github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -604,23 +624,18 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -647,18 +662,15 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0= github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -668,7 +680,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -678,7 +689,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -714,7 +724,6 @@ github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03O github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= @@ -748,28 +757,32 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= -go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI= -go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= -go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= -go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= -go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= -go.etcd.io/etcd/pkg/v3 v3.5.5/go.mod h1:6ksYFxttiUGzC2uxyqiyOEvhAiD0tuIqSZkX3TyPdaE= -go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= -go.etcd.io/etcd/raft/v3 v3.5.5/go.mod h1:76TA48q03g1y1VpTue92jZLr9lIHKUNcYdZOOGyx8rI= -go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= -go.etcd.io/etcd/server/v3 v3.5.5/go.mod h1:rZ95vDw/jrvsbj9XpTqPrTAB9/kzchVdhRirySPkUBc= +go.etcd.io/etcd/client/v2 v2.305.15 h1:VG2xbf8Vz1KJh65Ar2V5eDmfkp1bpzkSEHlhJM3usp8= +go.etcd.io/etcd/client/v2 v2.305.15/go.mod h1:Ad5dRjPVb/n5yXgAWQ/hXzuXXkBk0Y658ocuXYaUU48= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.etcd.io/etcd/pkg/v3 v3.5.15 h1:/Iu6Sr3iYaAjy++8sIDoZW9/EfhcwLZwd4FOZX2mMOU= +go.etcd.io/etcd/pkg/v3 v3.5.15/go.mod h1:e3Acf298sPFmTCGTrnGvkClEw9RYIyPtNzi1XM8rets= +go.etcd.io/etcd/raft/v3 v3.5.15 h1:jOA2HJF7zb3wy8H/pL13e8geWqkEa/kUs0waUggZC0I= +go.etcd.io/etcd/raft/v3 v3.5.15/go.mod h1:k3r7P4seEiUcgxOPLp+mloJWV3Q4QLPGNvy/OgC8OtM= +go.etcd.io/etcd/server/v3 v3.5.15 h1:x35jrWnZgsRwMsFsUJIUdT1bvzIz1B+29HjMfRYVN/E= +go.etcd.io/etcd/server/v3 v3.5.15/go.mod h1:l9jX9oa/iuArjqz0RNX/TDbc70dLXxRZo/nmPucrpFo= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -777,32 +790,25 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0 h1:g/BAN5o90Pr6D8xMRezjzGOHBpc15U+4oE53nZLiae4= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0/go.mod h1:+F41JBSkye7aYJELRvIMF0Z66reIwIOL0St75ZVwSJs= -go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.13.0 h1:1ZAKnNQKwBBxFtww/GwxNUyTf0AxkZzrukO8MeXqe4Y= -go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 h1:pa05sNT/P8OsIQ8mPZKTIyiBuzS/xDGLVx+DCt0y6Vs= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0/go.mod h1:rqbht/LlhVBgn5+k3M5QK96K5Xb0DvXpMJ5SFQpY6uw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 h1:Any/nVxaoMq1T2w0W85d6w5COlLuCCgOYKQhJJWEMwQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0/go.mod h1:46vAP6RWfNn7EKov73l5KBFlNxz8kYlxR1woU+bJ4ZY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0 h1:Wz7UQn7/eIqZVDJbuNEM6PmqeA71cWXrWcXekP5HZgU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0/go.mod h1:OhH1xvgA5jZW2M/S4PcvtDlFE1VULRRBsibBrKuJQGI= -go.opentelemetry.io/otel/metric v0.35.0 h1:aPT5jk/w7F9zW51L7WgRqNKDElBdyRLGuBtI5MX34e8= -go.opentelemetry.io/otel/metric v0.35.0/go.mod h1:qAcbhaTRFU6uG8QM7dDo7XvFsWcugziq/5YI065TokQ= -go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= -go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5JaM= -go.opentelemetry.io/otel/sdk v1.13.0/go.mod h1:YLKPx5+6Vx/o1TCUYYs+bpymtkmazOMT6zoRrC7AQ7I= -go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY= -go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= @@ -810,8 +816,8 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= @@ -833,7 +839,7 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -873,6 +879,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -923,6 +932,7 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -937,9 +947,8 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -951,14 +960,15 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -978,7 +988,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1000,7 +1009,6 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1016,7 +1024,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1030,9 +1037,10 @@ golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= @@ -1046,18 +1054,16 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1114,10 +1120,17 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1140,6 +1153,8 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1192,12 +1207,11 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -1220,14 +1234,11 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= -google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQGivcXdTuY3Y9NBvr70X+y77woofSRluec= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1262,7 +1273,6 @@ gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/R gopkg.in/httprequest.v1 v1.2.1/go.mod h1:x2Otw96yda5+8+6ZeWwHIJTFkEHWP/qP8pJOzqEtWPM= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -1270,12 +1280,10 @@ gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3M gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1303,6 +1311,5 @@ k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMei rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/core/reader/target_client.go b/core/reader/target_client.go index 9a67ac31..befc567d 100644 --- a/core/reader/target_client.go +++ b/core/reader/target_client.go @@ -23,6 +23,7 @@ import ( "context" "database/sql" "database/sql/driver" + "github.com/elastic/go-elasticsearch/v8" "github.com/milvus-io/milvus-sdk-go/v2/entity" "google.golang.org/api/iterator" "strings" @@ -48,14 +49,15 @@ type TargetClient struct { } type TargetConfig struct { - TargetDBType string - Address string - Username string - Password string - ProjectId string - APIKey string - EnableTLS bool - DialConfig util.DialConfig + TargetDBType string + Address string + Username string + Password string + ProjectId string + APIKey string + EnableTLS bool + ConnectionTimeout int + DialConfig util.DialConfig } func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) { @@ -81,6 +83,12 @@ func NewTarget(ctx context.Context, config TargetConfig) (api.TargetAPI, error) log.Warn("fail to new target client", zap.String("address", config.ProjectId), zap.Error(err)) return nil, err } + } else if strings.ToLower(targetClient.config.TargetDBType) == "elasticsearch" { + _, err := targetClient.GetElasticSearch(ctx, "") + if err != nil { + log.Warn("fail to new target client", zap.String("address", config.ProjectId), zap.Error(err)) + return nil, err + } } log.Info("new target client", zap.String("address", config.Address), zap.String("targetDBType", config.TargetDBType), zap.Any("config", targetClient.config)) @@ -101,10 +109,15 @@ func (t *TargetClient) GetMilvus(ctx context.Context, databaseName string) (clie func (t *TargetClient) GetMySQL(ctx context.Context, databaseName string) (*sql.DB, error) { apiKey := t.config.APIKey + if apiKey == "" { apiKey = util.GetAPIKey(t.config.Username, t.config.Password) + } else { + t.config.Username = strings.Split(apiKey, ":")[0] + t.config.Password = strings.Split(apiKey, ":")[1] } - mysqlClient, err := util.GetMySqlClientManager().GetMySQLClient(ctx, t.config.Address, apiKey, databaseName, t.config.EnableTLS, t.config.DialConfig) + + mysqlClient, err := util.GetMySqlClientManager().GetMySQLClient(ctx, t.config.Address, t.config.Username, t.config.Password, databaseName, t.config.EnableTLS, t.config.ConnectionTimeout) if err != nil { return nil, err } @@ -119,6 +132,14 @@ func (t *TargetClient) GetBigQuery(ctx context.Context) (*bigquery.Client, error return bigqueryClient, nil } +func (t *TargetClient) GetElasticSearch(ctx context.Context, index string) (*elasticsearch.Client, error) { + esClient, err := util.GetBigQueryClientManager().GetESClient(ctx, t.config.Address, t.config.Username, t.config.Password, index, t.config.EnableTLS, t.config.ConnectionTimeout) + if err != nil { + return nil, err + } + return esClient, nil +} + func (t *TargetClient) GetCollectionInfo(ctx context.Context, targetDBType, collectionName, databaseName string) (*model.CollectionInfo, error) { databaseName, err := t.GetDatabaseName(ctx, collectionName, databaseName) if err != nil { diff --git a/core/util/elasticsearch_client_resource.go b/core/util/elasticsearch_client_resource.go new file mode 100644 index 00000000..ae5d57c8 --- /dev/null +++ b/core/util/elasticsearch_client_resource.go @@ -0,0 +1,104 @@ +/* + * Licensed to the LF AI & Data foundation under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * // + * http://www.apache.org/licenses/LICENSE-2.0 + * // + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "context" + "crypto/tls" + "fmt" + "github.com/elastic/go-elasticsearch/v8" + "github.com/milvus-io/milvus/pkg/util/resource" + "github.com/zilliztech/milvus-cdc/core/log" + "go.uber.org/zap" + "net/http" + "time" +) + +const ( + ESClientResourceTyp = "es_client" + ESClientExpireTime = 30 * time.Second + DefaultESIndexName = "default-index" +) + +type resourceManager struct{} + +func (rm *resourceManager) Get(resourceType, resourceName string, createFunc func() (resource.Resource, error)) (resource.Resource, error) { + // Implement resource retrieval logic + return nil, nil +} + +// newMySQLClient(address, user, passwd, database string, enableTLS bool, connectionTimeout int) resource.NewResourceFunc +func (m *ClientResourceManager) newESClient(ctx context.Context, address, user, password, index string, enableTLS bool, connectionTimeout int) func() (resource.Resource, error) { + return func() (resource.Resource, error) { + cfg := elasticsearch.Config{ + Addresses: []string{ + address, // "http://localhost:9200" + }, + Username: user, + Password: password, + // CertificateFingerprint : "SHA256", + // ServiceToken: "AAA", + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // 필요시 TLS 설정 + IdleConnTimeout: time.Duration(connectionTimeout) * time.Second, + }, + } + + es, err := elasticsearch.NewClient(cfg) + if err != nil { + log.Warn("fail to new the mysql client", zap.String("user", user), zap.String("address", address), zap.Error(err)) + + return nil, err + } + + res := resource.NewSimpleResource(es, "ElasticsearchClientResourceType", fmt.Sprintf("%s:%s", address, index), time.Hour, func() { + // No explicit close method, just let GC handle it + }) + + return res, nil + } +} + +func (m *ClientResourceManager) GetESClient(ctx context.Context, address, user, passwd, index string, enableTLS bool, connectionTimeout int) (*elasticsearch.Client, error) { + if index == "" { + index = DefaultESIndexName + } + + ctxLog := log.Ctx(ctx).With(zap.String("index", index), zap.String("address", address)) + + res, err := m.manager.Get("ElasticsearchClientResourceType", + getESClientResourceName(address, index), + m.newESClient(ctx, address, user, passwd, index, enableTLS, connectionTimeout)) + if err != nil { + ctxLog.Error("fail to get Elasticsearch client", zap.Error(err)) + + return nil, err + } + if obj, ok := res.Get().(*elasticsearch.Client); ok && obj != nil { + return obj, nil + } + + ctxLog.Error("invalid resource object", zap.Any("obj", res.Get())) + + return nil, fmt.Errorf("invalid resource object") +} + +func getESClientResourceName(address, index string) string { + return fmt.Sprintf("%s:%s", address, index) +} diff --git a/core/util/mysql_client_resource.go b/core/util/mysql_client_resource.go index 5844fcf6..0edb0d62 100644 --- a/core/util/mysql_client_resource.go +++ b/core/util/mysql_client_resource.go @@ -24,7 +24,6 @@ import ( "fmt" "github.com/go-sql-driver/mysql" "reflect" - "strings" "time" "github.com/cockroachdb/errors" @@ -37,18 +36,20 @@ import ( const ( MySQLClientResourceTyp = "mysql_client" - MySQLClientExpireTime = 30 * time.Second DefaultMySQLDbName = "information_schema" - connectionTimeout = 1 + ConnectionTimeout = 30 + MySQLClientExpireTime = ConnectionTimeout * time.Second ) -func (m *ClientResourceManager) newMySQLClient(ctx context.Context, address, apiKey, database string, enableTLS bool, dialConfig DialConfig) resource.NewResourceFunc { +func (m *ClientResourceManager) newMySQLClient(address, user, passwd, database string, enableTLS bool, connectionTimeout int) resource.NewResourceFunc { return func() (resource.Resource, error) { - + if connectionTimeout == 0 { + connectionTimeout = ConnectionTimeout + } cfg := mysql.Config{ Addr: address, - User: strings.Split(apiKey, ":")[0], - Passwd: strings.Split(apiKey, ":")[1], + User: user, + Passwd: passwd, Net: "tcp", AllowNativePasswords: true, Timeout: time.Duration(connectionTimeout) * time.Second, @@ -68,7 +69,7 @@ func (m *ClientResourceManager) newMySQLClient(ctx context.Context, address, api } } -func (m *ClientResourceManager) GetMySQLClient(ctx context.Context, address, apiKey, database string, enableTLS bool, dialConfig DialConfig) (*sql.DB, error) { +func (m *ClientResourceManager) GetMySQLClient(ctx context.Context, address, user, passwd, database string, enableTLS bool, connectionTimeout int) (*sql.DB, error) { if database == "" { database = DefaultMySQLDbName } @@ -76,7 +77,7 @@ func (m *ClientResourceManager) GetMySQLClient(ctx context.Context, address, api ctxLog := log.Ctx(ctx).With(zap.String("database", database), zap.String("address", address)) res, err := m.manager.Get(MySQLClientResourceTyp, getMySQLClientResourceName(address, database), - m.newMySQLClient(ctx, address, apiKey, database, enableTLS, dialConfig)) + m.newMySQLClient(address, user, passwd, database, enableTLS, connectionTimeout)) if err != nil { ctxLog.Error("fail to get mysql client", zap.Error(err)) return nil, err diff --git a/core/writer/bigquery_handler.go b/core/writer/bigquery_handler.go index cda140e9..2bd9e07a 100644 --- a/core/writer/bigquery_handler.go +++ b/core/writer/bigquery_handler.go @@ -105,34 +105,6 @@ func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, f func(bigqueryCli return retryBigQueryFunc(bigqueryClient) } -/* - func (m *BigQueryDataHandler) bigqueryOp(ctx context.Context, query string, params map[string]interface{}) error { - retryFunc := func() error { - - q := m.client.Query(query) - - job, err := q.Run(ctx) - if err != nil { - log.Warn("failed to run query", zap.Error(err)) - return err - } - status, err := job.Wait(ctx) - if err != nil { - return err - } - if err := status.Err(); err != nil { - return err - } - return nil - } - - err := backoff.Retry(retryFunc, backoff.WithContext(m.retryOptions, ctx)) - if err != nil { - log.Warn("retry operation failed", zap.Error(err)) - } - return err - } -*/ func (m *BigQueryDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { schema := bigquery.Schema{} for _, field := range param.Schema.Fields { diff --git a/core/writer/es_handler.go b/core/writer/es_handler.go index 682ad26c..515c2971 100644 --- a/core/writer/es_handler.go +++ b/core/writer/es_handler.go @@ -1,14 +1,17 @@ package writer import ( + "bytes" "context" - "database/sql" + "encoding/json" "fmt" - "github.com/go-sql-driver/mysql" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/pkg/util/retry" + "github.com/zilliztech/milvus-cdc/core/util" "time" - "github.com/cenkalti/backoff/v4" "github.com/cockroachdb/errors" "github.com/zilliztech/milvus-cdc/core/api" "github.com/zilliztech/milvus-cdc/core/config" @@ -23,18 +26,16 @@ type ESDataHandler struct { username string password string connectTimeout int - db *sql.DB - retryOptions *backoff.ExponentialBackOff - mysqlCli *sql.DB + retryOptions []retry.Option + esClient *elasticsearch.Client } -// NewBigQueryDataHandler options must include ProjectIDOption and CredentialsOption func NewESDataHandler(options ...config.Option[*ESDataHandler]) (*ESDataHandler, error) { handler := &ESDataHandler{ connectTimeout: 5, - retryOptions: backoff.NewExponentialBackOff(), } - handler.retryOptions.MaxElapsedTime = 2 * time.Minute // 설정된 최대 재시도 시간 + + handler.retryOptions = util.GetRetryOptions(config.GetCommonConfig().Retry) for _, option := range options { option.Apply(handler) @@ -43,70 +44,99 @@ func NewESDataHandler(options ...config.Option[*ESDataHandler]) (*ESDataHandler, return nil, errors.New("empty MySQL address") } + if handler.username == "" || handler.password == "" { + return nil, errors.New("empty MySQL username or password") + } + var err error + timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) + defer cancel() - handler.db, err = handler.createESClient(handler.connectTimeout) + err = handler.ESOp(timeoutContext, func(esClient *elasticsearch.Client) error { + return nil + }) if err != nil { - log.Error("failed to create mysql connection", zap.Error(err)) + log.Warn("fail to new the elastic search client", zap.Error(err)) return nil, err } + handler.retryOptions = util.GetRetryOptions(config.GetCommonConfig().Retry) return handler, nil } -func (m *ESDataHandler) createESClient(connectionTimeout int) (*sql.DB, error) { - cfg := mysql.Config{ - Addr: m.address, - User: m.username, - Passwd: m.password, - Net: "tcp", - AllowNativePasswords: true, - Timeout: time.Duration(connectionTimeout) * time.Second, +func (m *ESDataHandler) ESOp(ctx context.Context, f func(esClient *elasticsearch.Client) error) error { + retryMySqlFunc := func(c *elasticsearch.Client) error { + // TODO Retryable and non-retryable errors should be distinguished + var err error + retryErr := retry.Do(ctx, func() error { + err = f(c) + return err + }, m.retryOptions...) + if retryErr != nil && err != nil { + return err + } + if retryErr != nil { + return retryErr + } + return nil } - db, err := sql.Open("mysql", cfg.FormatDSN()) + esClient, err := util.GetMySqlClientManager().GetESClient(ctx, m.address, m.username, m.password, "", false, m.connectTimeout) if err != nil { - log.Info("failed to connect mysql database", zap.Any("info", cfg.FormatDSN())) - return nil, fmt.Errorf("failed to connect mysql database : %v", err) + log.Warn("fail to get elastic search client", zap.Error(err)) + return err } - return db, nil + return retryMySqlFunc(esClient) +} +func (m *ESDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { + return m.ESOp(ctx, func(es *elasticsearch.Client) error { + _, err := es.Indices.Create(param.Schema.CollectionName) + return err + }) } -func (m *ESDataHandler) ESOp(ctx context.Context, query string, args ...interface{}) error { - retryFunc := func() error { - log.Info("executing mysql operation", zap.String("query", query), zap.Any("args", args)) - _, err := m.db.Exec(query) - if err != nil { - log.Warn("failed to execute mysql operation", zap.Error(err)) - } +func (m *ESDataHandler) DropCollection(ctx context.Context, param *api.DropCollectionParam) error { + return m.ESOp(ctx, func(es *elasticsearch.Client) error { + _, err := es.Indices.Delete([]string{param.CollectionName}) return err + }) + +} + +func insertDocument(ctx context.Context, es *elasticsearch.Client, indexName string, docs []map[string]interface{}) error { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(docs); err != nil { + return fmt.Errorf("Error encoding document: %v", err) + } + + req := esapi.IndexRequest{ + Index: indexName, + Body: &buf, + // DocumentID: fmt.Sprintf("%v", doc["id"]), + Refresh: "true", // Refresh the index after performing the request } - err := backoff.Retry(retryFunc, backoff.WithContext(m.retryOptions, ctx)) + res, err := req.Do(ctx, es) if err != nil { - log.Warn("retry operation failed", zap.Error(err)) + return fmt.Errorf("Error indexing document: %v", err) } - return err -} + defer res.Body.Close() -func (m *ESDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { - query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", param.Schema.CollectionName, param.Schema.Fields) - log.Info("CREATE TABLE", zap.String("query", query)) - return m.ESOp(ctx, query) -} + if res.IsError() { + return fmt.Errorf("Error indexing document Name=%s: %s", indexName, res.String()) + } -func (m *ESDataHandler) DropCollection(ctx context.Context, param *api.DropCollectionParam) error { - query := fmt.Sprintf("DROP TABLE IF EXISTS %s", param.CollectionName) - log.Info("DROP TABLE", zap.String("query", query)) - return m.ESOp(ctx, query) + log.Info("Document inserted successfully") + + return nil } func (m *ESDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { - columns := []string{} + var columns []string - rowValues := [][]interface{}{} + var rowValues [][]interface{} for _, col := range param.Columns { columns = append(columns, fmt.Sprintf("`%s`", col.Name())) @@ -165,7 +195,7 @@ func (m *ESDataHandler) Insert(ctx context.Context, param *api.InsertParam) erro vec = append(vec, v) if cnt == dim { - colValues = append(colValues, fmt.Sprintf("string_to_vector('[%v]')", join(float32SliceToStringSlice(vec), ","))) + colValues = append(colValues, vec) vec = []float32{} cnt = 1 } else { @@ -182,95 +212,107 @@ func (m *ESDataHandler) Insert(ctx context.Context, param *api.InsertParam) erro } } - var value, values string + var docs []map[string]interface{} + + doc := make(map[string]interface{}) for rowCnt := 0; rowCnt < len(rowValues[0]); rowCnt++ { - for colNo, _ := range columns { + for colNo, colName := range columns { switch rowValues[colNo][rowCnt].(type) { case string: - if colNo == 0 { - value = fmt.Sprintf("(%s", fmt.Sprintf("%s", rowValues[colNo][rowCnt])) - } else { - value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("%s", rowValues[colNo][rowCnt])) - } + doc[colName] = fmt.Sprintf("'%v'", rowValues[colNo][rowCnt]) default: - if colNo == 0 { - value = fmt.Sprintf("(%s", fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) - } else { - value = fmt.Sprintf("%s,%s", value, fmt.Sprintf("'%v'", rowValues[colNo][rowCnt])) - } + doc[colName] = rowValues[colNo][rowCnt] } } - value = fmt.Sprintf("%s)", value) + docs = append(docs, doc) + } - if rowCnt == 0 { - values = fmt.Sprintf("%s", value) - } else { - values = fmt.Sprintf("%s,%s", values, value) - } + return m.ESOp(ctx, func(es *elasticsearch.Client) error { + return insertDocument(ctx, es, param.CollectionName, docs) + }) +} + +func deleteDocumentsByQuery(ctx context.Context, es *elasticsearch.Client, index string, query map[string]interface{}) error { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(query); err != nil { + return fmt.Errorf("Error encoding query: %v", err) + } + + // Delete By Query 요청 생성 + req := esapi.DeleteByQueryRequest{ + Index: []string{index}, + Body: &buf, } - //string_to_vector('[1,2,3]') - query := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s", - param.Database, param.CollectionName, join(columns, ","), values) - // log.Info("INSERT", zap.String("query", query)) + // 요청 실행 + res, err := req.Do(ctx, es) + if err != nil { + return fmt.Errorf("Error executing request: %v", err) + } + defer res.Body.Close() + + if res.IsError() { + return fmt.Errorf("Error response from server: %s", res.String()) + } - return m.ESOp(ctx, query, nil) + log.Info("Documents deleted successfully") + return nil } func (m *ESDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { - query := fmt.Sprintf("DELETE FROM `%s.%s` WHERE %s = @value", param.Database, param.CollectionName, param.Column.Name()) - params := map[string]interface{}{"value": param.Column.FieldData()} - return m.ESOp(ctx, query, params) + query := map[string]interface{}{ + "query": map[string]interface{}{ + "term": map[string]interface{}{ + param.Column.Name(): param.Column.FieldData(), + }, + }, + } + + res, err := m.esClient.Delete(param.CollectionName, param.Column.FieldData().String()) + if err != nil { + return fmt.Errorf("Error getting response: %v", err) + } + defer res.Body.Close() + + if res.IsError() { + return fmt.Errorf("Error deleting document ID=%s: %s", param.Column.FieldData().String(), res.String()) + } + + log.Info("Document ID=%s deleted successfully", zap.String("doc-id", param.Column.FieldData().String())) + + return m.ESOp(ctx, func(es *elasticsearch.Client) error { + return deleteDocumentsByQuery(ctx, es, param.CollectionName, query) + }) } func (m *ESDataHandler) CreateIndex(ctx context.Context, param *api.CreateIndexParam) error { - // BigQuery는 Vector Search Index만을 지원합니다. - query := fmt.Sprintf("CREATE OR REPLACE VECTOR INDEX `%s`.%s ON `%s`(embedding_v1) OPTIONS(distance_type='L2',index_type_type='IVF')", param.Database, param.IndexName, param.CollectionName) - return m.ESOp(ctx, query, nil) + log.Warn("CreateIndex is not implemented in ElasticSearch, please check it") + return nil } func (m *ESDataHandler) DropIndex(ctx context.Context, param *api.DropIndexParam) error { // BigQuery는 인덱스를 지원하지 않습니다. + log.Warn("DropIndex is not implemented in ElasticSearch, please check it") return nil } func (m *ESDataHandler) CreateDatabase(ctx context.Context, param *api.CreateDatabaseParam) error { - query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", param.DbName) - log.Info("CREATE DATABASE", zap.String("query", query)) - return m.ESOp(ctx, query) + log.Warn("CreateDatabase is not implemented in ElasticSearch, please check it") + return nil } func (m *ESDataHandler) DropDatabase(ctx context.Context, param *api.DropDatabaseParam) error { - query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", param.DbName) - log.Info("DROP DATABASE", zap.String("query", query)) - return m.ESOp(ctx, query) + log.Warn("DropDatabase is not implemented in ElasticSearch, please check it") + return nil } func (m *ESDataHandler) DescribeCollection(ctx context.Context, param *api.DescribeCollectionParam) error { - query := fmt.Sprintf("DESCRIBE %s", param.Name) - log.Info("DESCRIBE", zap.String("query", query)) - return m.ESOp(ctx, query) + log.Warn("DescribeCollection is not implemented in ElasticSearch, please check it") + return nil } func (m *ESDataHandler) DescribeDatabase(ctx context.Context, param *api.DescribeDatabaseParam) error { - query := "SHOW DATABASES" - log.Info(query) - rows, err := m.db.QueryContext(ctx, query) - if err != nil { - return err - } - defer rows.Close() - - var dbName string - for rows.Next() { - err := rows.Scan(&dbName) - if err != nil { - return err - } - if dbName == param.Name { - return nil - } - } - return errors.Newf("database [%s] not found", param.Name) + log.Warn("DescribeDatabase is not implemented in ElasticSearch, please check it") + return nil } diff --git a/core/writer/mysql_handler.go b/core/writer/mysql_handler.go index 0e294348..2ec39c87 100644 --- a/core/writer/mysql_handler.go +++ b/core/writer/mysql_handler.go @@ -4,11 +4,11 @@ import ( "context" "database/sql" "fmt" - "github.com/cenkalti/backoff/v4" - "github.com/go-sql-driver/mysql" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/mq/msgstream" + "github.com/milvus-io/milvus/pkg/util/retry" + "github.com/zilliztech/milvus-cdc/core/util" "strconv" "strings" "time" @@ -30,16 +30,16 @@ type MySQLDataHandler struct { password string connectTimeout int db *sql.DB - retryOptions *backoff.ExponentialBackOff + retryOptions []retry.Option mysqlCli *sql.DB } func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDataHandler, error) { handler := &MySQLDataHandler{ connectTimeout: 5, - retryOptions: backoff.NewExponentialBackOff(), } - handler.retryOptions.MaxElapsedTime = 2 * time.Minute // 설정된 최대 재시도 시간 + + handler.retryOptions = util.GetRetryOptions(config.GetCommonConfig().Retry) for _, option := range options { option.Apply(handler) @@ -48,63 +48,70 @@ func NewMySQLDataHandler(options ...config.Option[*MySQLDataHandler]) (*MySQLDat return nil, errors.New("empty MySQL address") } + if handler.username == "" || handler.password == "" { + return nil, errors.New("empty MySQL username or password") + } + var err error + timeoutContext, cancel := context.WithTimeout(context.Background(), time.Duration(handler.connectTimeout)*time.Second) + defer cancel() - err = handler.createDBConnection(handler.connectTimeout) + err = handler.mysqlOp(timeoutContext, func(mysqlClient *sql.DB) error { + return nil + }) if err != nil { - log.Error("failed to create mysql connection", zap.Error(err)) + log.Warn("fail to new the milvus client", zap.Error(err)) return nil, err } + handler.retryOptions = util.GetRetryOptions(config.GetCommonConfig().Retry) return handler, nil } -func (m *MySQLDataHandler) createDBConnection(connectionTimeout int) (err error) { - cfg := mysql.Config{ - Addr: m.address, - User: m.username, - Passwd: m.password, - Net: "tcp", - AllowNativePasswords: true, - Timeout: time.Duration(connectionTimeout) * time.Second, +func (m *MySQLDataHandler) mysqlOp(ctx context.Context, f func(mysqlClient *sql.DB) error) error { + retryMySqlFunc := func(c *sql.DB) error { + // TODO Retryable and non-retryable errors should be distinguished + var err error + retryErr := retry.Do(ctx, func() error { + err = f(c) + return err + }, m.retryOptions...) + if retryErr != nil && err != nil { + return err + } + if retryErr != nil { + return retryErr + } + return nil } - m.db, err = sql.Open("mysql", cfg.FormatDSN()) + mysqlClient, err := util.GetMySqlClientManager().GetMySQLClient(ctx, m.address, m.username, m.password, "", false, m.connectTimeout) if err != nil { - log.Info("failed to connect mysql database", zap.Any("info", cfg.FormatDSN())) - return fmt.Errorf("failed to connect mysql database : %v", err) - } - - return nil -} - -func (m *MySQLDataHandler) mysqlOp(ctx context.Context, query string, args ...interface{}) error { - retryFunc := func() error { - log.Info("executing mysql operation", zap.String("query", query), zap.Any("args", args)) - _, err := m.db.Exec(query) - if err != nil { - log.Warn("failed to execute mysql operation", zap.Error(err)) - } + log.Warn("fail to get bigquery client", zap.Error(err)) return err } - err := backoff.Retry(retryFunc, backoff.WithContext(m.retryOptions, ctx)) - if err != nil { - log.Warn("retry operation failed", zap.Error(err)) - } - return err + return retryMySqlFunc(mysqlClient) } func (m *MySQLDataHandler) CreateCollection(ctx context.Context, param *api.CreateCollectionParam) error { query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", param.Schema.CollectionName, param.Schema.Fields) log.Info("CREATE TABLE", zap.String("query", query)) - return m.mysqlOp(ctx, query) + + return m.mysqlOp(ctx, func(mysqlClient *sql.DB) error { + _, err := mysqlClient.Exec(query) + return err + }) } func (m *MySQLDataHandler) DropCollection(ctx context.Context, param *api.DropCollectionParam) error { query := fmt.Sprintf("DROP TABLE IF EXISTS %s", param.CollectionName) log.Info("DROP TABLE", zap.String("query", query)) - return m.mysqlOp(ctx, query) + + return m.mysqlOp(ctx, func(mysqlClient *sql.DB) error { + _, err := mysqlClient.Exec(query) + return err + }) } func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) error { @@ -219,7 +226,10 @@ func (m *MySQLDataHandler) Insert(ctx context.Context, param *api.InsertParam) e param.Database, param.CollectionName, join(columns, ","), values) // log.Info("INSERT", zap.String("query", query)) - return m.mysqlOp(ctx, query) + return m.mysqlOp(ctx, func(mysqlClient *sql.DB) error { + _, err := mysqlClient.Exec(query) + return err + }) } func interfaceSliceToStringSlice(input []interface{}) []string { @@ -240,9 +250,13 @@ func float32SliceToStringSlice(input []float32) []string { } func (m *MySQLDataHandler) Delete(ctx context.Context, param *api.DeleteParam) error { - query := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE `%s` = ?", param.Database, param.CollectionName, param.Column.Name) + query := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE `%s` = '%s'", param.Database, param.CollectionName, param.Column.Name, param.Column.FieldData()) log.Info("DELETE", zap.String("query", query)) - return m.mysqlOp(ctx, query, param.Column.FieldData()) + + return m.mysqlOp(ctx, func(mysqlClient *sql.DB) error { + _, err := mysqlClient.Exec(query) + return err + }) } func (m *MySQLDataHandler) CreatePartition(ctx context.Context, param *api.CreatePartitionParam) error { @@ -259,13 +273,21 @@ func (m *MySQLDataHandler) CreateIndex(ctx context.Context, param *api.CreateInd query := fmt.Sprintf("CREATE INDEX %s ON `%s`.`%s` (`%s`)", param.GetIndexName(), param.DbName, param.GetCollectionName(), param.GetFieldName()) log.Info("CREATE INDEX", zap.String("query", query)) - return m.mysqlOp(ctx, query) + + return m.mysqlOp(ctx, func(mysqlClient *sql.DB) error { + _, err := mysqlClient.Exec(query) + return err + }) } func (m *MySQLDataHandler) DropIndex(ctx context.Context, param *api.DropIndexParam) error { query := fmt.Sprintf("DROP INDEX %s ON %s", param.IndexName, param.CollectionName) log.Info("DROP INDEX", zap.String("query", query)) - return m.mysqlOp(ctx, query) + + return m.mysqlOp(ctx, func(mysqlClient *sql.DB) error { + _, err := mysqlClient.Exec(query) + return err + }) } func (m *MySQLDataHandler) LoadCollection(ctx context.Context, param *api.LoadCollectionParam) error { @@ -295,13 +317,21 @@ func (m *MySQLDataHandler) Flush(ctx context.Context, param *api.FlushParam) err func (m *MySQLDataHandler) CreateDatabase(ctx context.Context, param *api.CreateDatabaseParam) error { query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", param.DbName) log.Info("CREATE DATABASE", zap.String("query", query)) - return m.mysqlOp(ctx, query) + + return m.mysqlOp(ctx, func(mysqlClient *sql.DB) error { + _, err := mysqlClient.Exec(query) + return err + }) } func (m *MySQLDataHandler) DropDatabase(ctx context.Context, param *api.DropDatabaseParam) error { query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", param.DbName) log.Info("DROP DATABASE", zap.String("query", query)) - return m.mysqlOp(ctx, query) + + return m.mysqlOp(ctx, func(mysqlClient *sql.DB) error { + _, err := mysqlClient.Exec(query) + return err + }) } func (m *MySQLDataHandler) unmarshalTsMsg(ctx context.Context, msgType commonpb.MsgType, dbName string, msgBytes []byte) error { @@ -440,7 +470,6 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl log.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err)) return err } - } return nil @@ -449,7 +478,11 @@ func (m *MySQLDataHandler) ReplicateMessage(ctx context.Context, param *api.Repl func (m *MySQLDataHandler) DescribeCollection(ctx context.Context, param *api.DescribeCollectionParam) error { query := fmt.Sprintf("DESCRIBE %s", param.Name) log.Info("DESCRIBE", zap.String("query", query)) - return m.mysqlOp(ctx, query) + + return m.mysqlOp(ctx, func(mysqlClient *sql.DB) error { + _, err := mysqlClient.Exec(query) + return err + }) } func (m *MySQLDataHandler) DescribeDatabase(ctx context.Context, param *api.DescribeDatabaseParam) error { diff --git a/server/go.mod b/server/go.mod index 5c7b2bbf..54be5131 100644 --- a/server/go.mod +++ b/server/go.mod @@ -7,7 +7,7 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/goccy/go-json v0.10.2 github.com/golang/protobuf v1.5.4 - github.com/google/uuid v1.4.0 + github.com/google/uuid v1.6.0 github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240604033341-0f2b1f29bf47 github.com/milvus-io/milvus/pkg v0.0.2-0.20240618133959-0d20303e54ae @@ -15,33 +15,30 @@ require ( github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/prometheus/client_golang v1.14.0 github.com/samber/lo v1.27.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/zilliztech/milvus-cdc/core v0.0.1 - go.etcd.io/etcd/client/v3 v3.5.5 + go.etcd.io/etcd/client/v3 v3.5.15 go.uber.org/zap v1.21.0 - google.golang.org/grpc v1.59.0 + google.golang.org/grpc v1.64.0 sigs.k8s.io/yaml v1.3.0 ) require ( - cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/bigquery v1.56.0 // indirect - cloud.google.com/go/compute v1.23.1 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.3 // indirect + cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go/bigquery v1.59.1 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v1.1.6 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect github.com/AthenZ/athenz v1.10.39 // indirect github.com/DataDog/zstd v1.5.0 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect - github.com/apache/arrow/go/v12 v12.0.0 // indirect + github.com/apache/arrow/go/v14 v14.0.2 // indirect github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 // indirect - github.com/apache/thrift v0.16.0 // indirect github.com/ardielle/ardielle-go v1.5.2 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cilium/ebpf v0.11.0 // indirect github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect @@ -55,43 +52,44 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect + github.com/elastic/go-elasticsearch/v8 v8.14.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/getsentry/sentry-go v0.12.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect - github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/googleapis/gax-go/v2 v2.12.2 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.7 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/linkedin/goavro/v2 v2.11.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect - github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/minio/highwayhash v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -106,17 +104,17 @@ require ( github.com/pelletier/go-toml v1.9.3 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pierrec/lz4 v2.5.2+incompatible // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/shirou/gopsutil/v3 v3.22.9 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.6.0 // indirect @@ -125,7 +123,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.8.1 // indirect github.com/streamnative/pulsarctl v0.5.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tidwall/gjson v1.14.4 // indirect @@ -139,44 +137,43 @@ require ( github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - go.etcd.io/etcd/api/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/v2 v2.305.5 // indirect - go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect - go.etcd.io/etcd/raft/v3 v3.5.5 // indirect - go.etcd.io/etcd/server/v3 v3.5.5 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/etcd/api/v3 v3.5.15 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect + go.etcd.io/etcd/client/v2 v2.305.15 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.15 // indirect + go.etcd.io/etcd/raft/v3 v3.5.15 // indirect + go.etcd.io/etcd/server/v3 v3.5.15 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0 // indirect - go.opentelemetry.io/otel v1.13.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0 // indirect - go.opentelemetry.io/otel/metric v0.35.0 // indirect - go.opentelemetry.io/otel/sdk v1.13.0 // indirect - go.opentelemetry.io/otel/trace v1.13.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.12.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.149.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/api v0.169.0 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/server/go.sum b/server/go.sum index d3b9f2a5..22d7e6f7 100644 --- a/server/go.sum +++ b/server/go.sum @@ -18,36 +18,27 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= -cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= -cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.56.0 h1:LHIc9E7Kw+ftFpQFKzZYBB88IAFz7qONawXXx0F3QBo= -cloud.google.com/go/bigquery v1.56.0/go.mod h1:KDcsploXTEY7XT3fDQzMUZlpQLHzE4itubHrnmhUrZA= -cloud.google.com/go/bigquery v1.57.1 h1:FiULdbbzUxWD0Y4ZGPSVCDLvqRSyCIO6zKV7E2nf5uA= -cloud.google.com/go/bigquery v1.57.1/go.mod h1:iYzC0tGVWt1jqSzBHqCr3lrRn0u13E8e+AqowBsDgug= -cloud.google.com/go/bigquery v1.62.0/go.mod h1:5ee+ZkF1x/ntgCsFQJAQTM3QkAZOecfCmvxhkJsWRSA= -cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= -cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= -cloud.google.com/go/compute v1.27.2/go.mod h1:YQuHkNEwP3bIz4LBYQqf4DIMfFtTDtnEgnwG0mJQQ9I= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/compute/metadata v0.4.0/go.mod h1:SIQh1Kkb4ZJ8zJ874fqVkslA29PRXuleyj6vOzlbK7M= -cloud.google.com/go/datacatalog v1.18.1 h1:xJp9mZrc2HPaoxIz3sP9pCmf/impifweQ/yGG9VBfio= -cloud.google.com/go/datacatalog v1.18.1/go.mod h1:TzAWaz+ON1tkNr4MOcak8EBHX7wIRX/gZKM+yTVsv+A= +cloud.google.com/go/bigquery v1.59.1 h1:CpT+/njKuKT3CEmswm6IbhNu9u35zt5dO4yPDLW+nG4= +cloud.google.com/go/bigquery v1.59.1/go.mod h1:VP1UJYgevyTwsV7desjzNzDND5p6hZB+Z8gZJN1GQUc= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/datacatalog v1.19.3 h1:A0vKYCQdxQuV4Pi0LL9p39Vwvg4jH5yYveMv50gU5Tw= +cloud.google.com/go/datacatalog v1.19.3/go.mod h1:ra8V3UAsciBpJKQ+z9Whkxzxv7jmQg1hfODr3N3YPJ4= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= -cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= -cloud.google.com/go/iam v1.1.10/go.mod h1:iEgMq62sg8zx446GCaijmA2Miwg5o3UbO+nI47WHJps= -cloud.google.com/go/longrunning v0.5.2 h1:u+oFqfEwwU7F9dIELigxbe0XVnBAo9wqMuQLA50CZ5k= -cloud.google.com/go/longrunning v0.5.2/go.mod h1:nqo6DQbNV2pXhGDbDMoN2bWz68MjZUzqv2YttZiveCs= +cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= +cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= +cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= +cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -57,8 +48,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= -cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg= +cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= @@ -75,10 +66,7 @@ github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= -github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/SimFG/milvus/pkg v0.0.0-20240628091152-f303bc30eaaa h1:YIuRENItb5LGPQy500gX977/TkUEtMUzfD/ujK4qJTg= github.com/SimFG/milvus/pkg v0.0.0-20240628091152-f303bc30eaaa/go.mod h1:XT4MPwUYb+UcvbBINVmqqcbePR3+KCEeBqYm6p2f/0A= @@ -91,13 +79,9 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc= -github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= -github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= +github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= @@ -114,18 +98,13 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -139,20 +118,13 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= @@ -162,30 +134,23 @@ github.com/confluentinc/confluent-kafka-go v1.9.1 h1:L3aW6KvTyrq/+BOMnDm9xJylhAE github.com/confluentinc/confluent-kafka-go v1.9.1/go.mod h1:ptXNqsuDfYbAE/LBW6pnwWZElUoWxHoV8E43DCrliyo= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= @@ -197,24 +162,26 @@ github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= +github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow= +github.com/elastic/go-elasticsearch/v8 v8.14.0/go.mod h1:WRvnlGkSuZyp83M2U8El/LGXpCjYLrvlkSgkAH4O5I4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= -github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= @@ -228,7 +195,6 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -250,9 +216,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= @@ -276,19 +241,15 @@ github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q8 github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -328,11 +289,9 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= -github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -378,16 +337,14 @@ github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= +github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= +github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -395,16 +352,14 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hamba/avro v1.5.6/go.mod h1:3vNT0RLXXpFm2Tb/5KC71ZRJlOroggq1Rcitb6k4Fr8= @@ -454,7 +409,6 @@ github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7 github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -477,20 +431,15 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= -github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -522,15 +471,15 @@ github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -554,10 +503,6 @@ github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240604033341-0f2b1f29bf47 h1:4T github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240604033341-0f2b1f29bf47/go.mod h1:6ckCQ8h5iFncZcaIpGE8T8sqliwaw2Eu9UzHaCURMA4= github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A= github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -601,7 +546,6 @@ github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -632,8 +576,7 @@ github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCr github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -655,7 +598,6 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:Om github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -667,31 +609,26 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -710,18 +647,15 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0= github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -731,7 +665,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -741,7 +674,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -749,8 +681,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -762,8 +694,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -779,7 +710,6 @@ github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03O github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= @@ -813,33 +743,31 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= -go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI= -go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= -go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= -go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= -go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= -go.etcd.io/etcd/pkg/v3 v3.5.5/go.mod h1:6ksYFxttiUGzC2uxyqiyOEvhAiD0tuIqSZkX3TyPdaE= -go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= -go.etcd.io/etcd/raft/v3 v3.5.5/go.mod h1:76TA48q03g1y1VpTue92jZLr9lIHKUNcYdZOOGyx8rI= -go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= -go.etcd.io/etcd/server/v3 v3.5.5/go.mod h1:rZ95vDw/jrvsbj9XpTqPrTAB9/kzchVdhRirySPkUBc= +go.etcd.io/etcd/client/v2 v2.305.15 h1:VG2xbf8Vz1KJh65Ar2V5eDmfkp1bpzkSEHlhJM3usp8= +go.etcd.io/etcd/client/v2 v2.305.15/go.mod h1:Ad5dRjPVb/n5yXgAWQ/hXzuXXkBk0Y658ocuXYaUU48= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.etcd.io/etcd/pkg/v3 v3.5.15 h1:/Iu6Sr3iYaAjy++8sIDoZW9/EfhcwLZwd4FOZX2mMOU= +go.etcd.io/etcd/pkg/v3 v3.5.15/go.mod h1:e3Acf298sPFmTCGTrnGvkClEw9RYIyPtNzi1XM8rets= +go.etcd.io/etcd/raft/v3 v3.5.15 h1:jOA2HJF7zb3wy8H/pL13e8geWqkEa/kUs0waUggZC0I= +go.etcd.io/etcd/raft/v3 v3.5.15/go.mod h1:k3r7P4seEiUcgxOPLp+mloJWV3Q4QLPGNvy/OgC8OtM= +go.etcd.io/etcd/server/v3 v3.5.15 h1:x35jrWnZgsRwMsFsUJIUdT1bvzIz1B+29HjMfRYVN/E= +go.etcd.io/etcd/server/v3 v3.5.15/go.mod h1:l9jX9oa/iuArjqz0RNX/TDbc70dLXxRZo/nmPucrpFo= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -849,37 +777,25 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0 h1:g/BAN5o90Pr6D8xMRezjzGOHBpc15U+4oE53nZLiae4= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0/go.mod h1:+F41JBSkye7aYJELRvIMF0Z66reIwIOL0St75ZVwSJs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.13.0 h1:1ZAKnNQKwBBxFtww/GwxNUyTf0AxkZzrukO8MeXqe4Y= -go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 h1:pa05sNT/P8OsIQ8mPZKTIyiBuzS/xDGLVx+DCt0y6Vs= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0/go.mod h1:rqbht/LlhVBgn5+k3M5QK96K5Xb0DvXpMJ5SFQpY6uw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 h1:Any/nVxaoMq1T2w0W85d6w5COlLuCCgOYKQhJJWEMwQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0/go.mod h1:46vAP6RWfNn7EKov73l5KBFlNxz8kYlxR1woU+bJ4ZY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0 h1:Wz7UQn7/eIqZVDJbuNEM6PmqeA71cWXrWcXekP5HZgU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0/go.mod h1:OhH1xvgA5jZW2M/S4PcvtDlFE1VULRRBsibBrKuJQGI= -go.opentelemetry.io/otel/metric v0.35.0 h1:aPT5jk/w7F9zW51L7WgRqNKDElBdyRLGuBtI5MX34e8= -go.opentelemetry.io/otel/metric v0.35.0/go.mod h1:qAcbhaTRFU6uG8QM7dDo7XvFsWcugziq/5YI065TokQ= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= -go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5JaM= -go.opentelemetry.io/otel/sdk v1.13.0/go.mod h1:YLKPx5+6Vx/o1TCUYYs+bpymtkmazOMT6zoRrC7AQ7I= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY= -go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -889,8 +805,8 @@ go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= @@ -914,10 +830,8 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -928,8 +842,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI= -golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -956,10 +869,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1008,12 +919,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1026,10 +934,8 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= -golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1041,17 +947,14 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1071,7 +974,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1093,7 +995,6 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1109,7 +1010,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1123,17 +1023,14 @@ golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1143,20 +1040,15 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1214,19 +1106,16 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= -gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1249,18 +1138,15 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= -google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= -google.golang.org/api v0.188.0/go.mod h1:VR0d+2SIiWOYG3r/jdm7adPW9hI2aRv9ETOSCQ9Beag= +google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= +google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1305,17 +1191,13 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= -google.golang.org/genproto v0.0.0-20240708141625-4ad9e859172b/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1336,15 +1218,10 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQGivcXdTuY3Y9NBvr70X+y77woofSRluec= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1361,8 +1238,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= @@ -1380,7 +1256,6 @@ gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/R gopkg.in/httprequest.v1 v1.2.1/go.mod h1:x2Otw96yda5+8+6ZeWwHIJTFkEHWP/qP8pJOzqEtWPM= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -1388,12 +1263,10 @@ gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3M gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1421,6 +1294,5 @@ k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMei rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/server/tool/msg_count/main.go b/server/tool/msg_count/main.go index a6b80d44..e5d93a75 100644 --- a/server/tool/msg_count/main.go +++ b/server/tool/msg_count/main.go @@ -43,7 +43,6 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/zilliztech/milvus-cdc/core/config" @@ -76,8 +75,8 @@ type PositionConfig struct { } func main() { - paramtable.Init() - paramtable.Get().Save(paramtable.Get().ServiceParam.MQCfg.EnablePursuitMode.Key, "false") + // paramtable.Init() + // paramtable.Get().Save(paramtable.Get().ServiceParam.MQCfg.EnablePursuitMode.Key, "false") log.ReplaceGlobals(zap.NewNop(), &log.ZapProperties{ Core: zapcore.NewNopCore(), Syncer: zapcore.AddSync(ioutil.Discard), From 9e6dc448a797cefccda14c41cfdbf9c703ecfd28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2024 07:36:07 +0000 Subject: [PATCH 118/118] Bump google.golang.org/grpc from 1.64.0 to 1.64.1 in /server Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.64.0 to 1.64.1. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.64.0...v1.64.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- server/go.mod | 2 +- server/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/go.mod b/server/go.mod index 54be5131..64079dce 100644 --- a/server/go.mod +++ b/server/go.mod @@ -19,7 +19,7 @@ require ( github.com/zilliztech/milvus-cdc/core v0.0.1 go.etcd.io/etcd/client/v3 v3.5.15 go.uber.org/zap v1.21.0 - google.golang.org/grpc v1.64.0 + google.golang.org/grpc v1.64.1 sigs.k8s.io/yaml v1.3.0 ) diff --git a/server/go.sum b/server/go.sum index 22d7e6f7..1a3e57a4 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1220,8 +1220,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQGivcXdTuY3Y9NBvr70X+y77woofSRluec= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=