From 834b67d2e1f23d5076cbe1dfe3df20483b2a3323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Thu, 20 Jun 2024 18:09:08 +0800 Subject: [PATCH 01/38] feat: add es --- cmd/api/main.go | 1 + cmd/api/option/option.go | 9 + db/elastic/dao/event.go | 556 ++++++++++++++++++++++++++++++++ db/elastic/dao/event_test.go | 24 ++ db/mysql/dao_impl.go | 8 +- pkg/component/core.go | 6 + pkg/component/es/esComponent.go | 88 +++++ 7 files changed, 687 insertions(+), 5 deletions(-) create mode 100644 db/elastic/dao/event.go create mode 100644 db/elastic/dao/event_test.go create mode 100644 pkg/component/es/esComponent.go diff --git a/cmd/api/main.go b/cmd/api/main.go index 8ee0b22950..9ef693ccb1 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -48,6 +48,7 @@ func main() { }) // 启动 rbd-api err := rainbond.New(context.Background(), configs.Default()). + Registry(component.ES()). Registry(component.Database()). Registry(component.Grpc()). Registry(component.Event()). diff --git a/cmd/api/option/option.go b/cmd/api/option/option.go index 71f935a534..de3c8918b4 100644 --- a/cmd/api/option/option.go +++ b/cmd/api/option/option.go @@ -65,6 +65,12 @@ type Config struct { GrctlImage string RbdHub string RbdWorker string + RegionName string + RegionSN string + + ElasticSearchURL string + ElasticSearchUsername string + ElasticSearchPassword string } // APIServer apiserver server @@ -121,6 +127,9 @@ func (a *APIServer) AddFlags(fs *pflag.FlagSet) { fs.StringSliceVar(&a.EventLogServers, "event-servers", []string{"rbd-eventlog:6366"}, "event log server address") fs.StringSliceVar(&a.EventLogEndpoints, "event-log", []string{"local=>rbd-eventlog:6363"}, "event log websocket address") + fs.StringVar(&a.ElasticSearchURL, "es-url", "127.0.0.1:9200", "es url") + fs.StringVar(&a.ElasticSearchUsername, "es-username", "", "es username") + fs.StringVar(&a.ElasticSearchPassword, "es-password", "", "es pwd") } // SetLog 设置log diff --git a/db/elastic/dao/event.go b/db/elastic/dao/event.go new file mode 100644 index 0000000000..c1d83f6b9f --- /dev/null +++ b/db/elastic/dao/event.go @@ -0,0 +1,556 @@ +package dao + +import ( + "context" + "encoding/json" + "fmt" + ctxutil "github.com/goodrain/rainbond/api/util/ctx" + "github.com/goodrain/rainbond/db" + "github.com/goodrain/rainbond/db/model" + "github.com/goodrain/rainbond/pkg/component/es" + "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" + "time" +) + +type EventDaoImpl struct { +} + +// AddModel AddModel +func (c *EventDaoImpl) AddModel(mo model.Interface) error { + result := mo.(*model.ServiceEvent) + body, _ := json.Marshal(result) + _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", result.EventID), string(body)) + if err != nil { + logrus.Errorf("eventDaoImpl addModel error: %s", err.Error()) + } + return err +} + +// UpdateModel UpdateModel +func (c *EventDaoImpl) UpdateModel(mo model.Interface) error { + update := mo.(*model.ServiceEvent) + body, _ := json.Marshal(update) + _, err := es.Default().PUT(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", update.EventID), string(body)) + return err +} + +// DeleteModel DeleteModel +func (c *EventDaoImpl) DeleteModel(id string, args ...interface{}) error { + _, err := es.Default().DELETE(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", id)) + return err +} + +// CreateEventsInBatch creates events in batch. +func (c *EventDaoImpl) CreateEventsInBatch(events []*model.ServiceEvent) error { + for _, event := range events { + _ = c.AddModel(event) + } + return nil +} + +// DeleteEvents delete event +func (c *EventDaoImpl) DeleteEvents(eventIDs []string) error { + eventIds, _ := json.Marshal(eventIDs) + query := fmt.Sprintf(` + { + "query": { + "terms": { + "event_id": %s + } + } + }`, string(eventIds)) + _, err := es.Default().POST("/appstore_tenant_services_event/_delete_by_query", query) + return err +} + +// UpdateReason update reasion. +func (c *EventDaoImpl) UpdateReason(eventID string, reason string) error { + body := fmt.Sprintf(`{ + "script": { + "source": "ctx._source.reason = params.reason", + "params": { + "reason": "%s" + } + }, + "query": { + "term": { + "event_id": "%s" + } + } +}`, reason, eventID) + _, err := es.Default().POST("/appstore_tenant_services_event/_update_by_query", body) + return err +} + +// GetEventByEventID get event log message +func (c *EventDaoImpl) GetEventByEventID(eventID string) (*model.ServiceEvent, error) { + var result model.ServiceEvent + + get, err := es.Default().GET(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", eventID)) + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(gjson.Get(get, "_source").Raw), &result) + if err != nil { + return nil, err + } + return &result, nil +} + +// GetEventByEventIDs get event info +func (c *EventDaoImpl) GetEventByEventIDs(eventIDs []string) ([]*model.ServiceEvent, error) { + eventIds, _ := json.Marshal(eventIDs) + query := fmt.Sprintf(` +{ + "query": { + "terms": { + "event_id": %s + } + } +}`, eventIds) + return c.array(query) +} + +func (c *EventDaoImpl) array(query string) ([]*model.ServiceEvent, error) { + get, err := es.Default().POST("/appstore_tenant_services_event/_search", query) + if err != nil { + return nil, err + } + var result []*model.ServiceEvent + err = json.Unmarshal([]byte(gjson.Get(get, "hits.hits").Raw), &result) + if err != nil { + return nil, err + } + return result, nil +} + +// UpdateInBatch - +func (c *EventDaoImpl) UpdateInBatch(events []*model.ServiceEvent) error { + for i := range events { + _ = c.UpdateModel(events[i]) + } + return nil +} + +// GetEventByServiceID get event log message +func (c *EventDaoImpl) GetEventByServiceID(serviceID string) ([]*model.ServiceEvent, error) { + body := fmt.Sprintf(`{ + "query": { + "match": { + "service_id": "%s" + } + }, + "sort": [ + { + "start_time": { + "order": "desc" + } + } + ] +}`, serviceID) + return c.array(body) +} + +// DelEventByServiceID delete event log +func (c *EventDaoImpl) DelEventByServiceID(serviceID string) error { + query := fmt.Sprintf(` + { + "query": { + "match": { + "service_id": "%s" + } + } + }`, serviceID) + _, err := es.Default().POST("/appstore_tenant_services_event/_delete_by_query", query) + return err +} + +// ListByTargetID - +func (c *EventDaoImpl) ListByTargetID(targetID string) ([]*model.ServiceEvent, error) { + query := fmt.Sprintf(` + { + "query": { + "match": { + "target_id": "%s" + } + } + }`, targetID) + return c.array(query) +} + +// GetEventsByTarget get event by target with page +func (c *EventDaoImpl) GetEventsByTarget(target, targetID string, offset, limit int) ([]*model.ServiceEvent, int, error) { + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "match": { "target": "%s" } }, + { "match": { "target_id": "%s" } } + ] + } + }, + "sort": [ + { "create_time": { "order": "desc" } }, + { "id": { "order": "desc" } } + ], + "from": %d, + "size": %d +} +`, target, targetID, offset, limit) + + array, err := c.array(body) + if err != nil { + return nil, 0, err + } + post, _ := es.Default().POST("/appstore_tenant_services_event/_count", body) + + return array, (int)(gjson.Get(post, "count").Int()), nil +} + +// GetEventsByTenantID get event by tenantID +func (c *EventDaoImpl) GetEventsByTenantID(tenantID string, offset, limit int) ([]*model.ServiceEvent, int, error) { + query := fmt.Sprintf(` + { + "query": { + "match": { + "tenant_id": "%s" + } + }, + "sort": [ + {"start_time": "desc"}, + {"id": "desc"} + ], + "from": %d, + "size": %d + }`, tenantID, offset, limit) + + post, _ := es.Default().POST("/appstore_tenant_services_event/_count", query) + count := (int)(gjson.Get(post, "count").Int()) + + array, err := c.array(query) + if err != nil { + return nil, 0, err + } + return array, count, nil +} + +// GetEventsByTenantIDs get my teams all event by tenantIDs +func (c *EventDaoImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit int) ([]*model.EventAndBuild, error) { + tenants, _ := json.Marshal(tenantIDs) + body := fmt.Sprintf(`{ + "sort": [ + { "id": "desc" } + ], + "from": %d, + "size": %d + "query": { + "bool": { + "must": [ + { + "term": { + "target": "service" + } + }, + { + "terms": { + "tenant_id": %s + } + } + ] + } + } +}`, offset, limit, string(tenants)) + + //ops, _ := json.Marshal(tenantIDs) + //// 使用原生 SQL 查询，并进行连接优化 + //query := ` + // SELECT + // a.ID, a.create_time, a.tenant_id, a.target, a.target_id, a.user_name, + // a.start_time, a.end_time, a.opt_type, a.syn_type, a.status, a.final_status, + // a.message, a.reason, b.build_version, b.kind, b.delivered_type, b.delivered_path, + // b.image_name, b.cmd, b.repo_url, b.code_version, b.code_branch, b.code_commit_msg, + // b.code_commit_author, b.plan_version + // FROM + // region.tenant_services_event AS a + // LEFT JOIN + // region.tenant_service_version AS b + // ON + // a.target_id = b.service_id AND a.event_id = b.event_id + // WHERE + // a.target = 'service' + // AND a.tenant_id IN (?) + // ORDER BY + // a.ID DESC + // LIMIT ?, ?; + //` + //if err := c.DB.Debug().Raw(query, tenantIDs, offset, limit).Scan(&events).Error; err != nil { + // return nil, err + //} + + array, err := c.array(body) + if err != nil { + return nil, err + } + res := make([]*model.EventAndBuild, 0) + for _, item := range array { + version, err := db.GetManager().VersionInfoDao().GetVersionByEventID(item.EventID) + if err != nil { + e := &model.EventAndBuild{ + CreateTime: item.CreatedAt.Format(time.DateTime), + TenantID: item.TenantID, + Target: item.Target, + TargetID: item.TargetID, + UserName: item.UserName, + StartTime: item.StartTime, + EndTime: item.EndTime, + OptType: item.OptType, + SynType: string(rune(item.SynType)), + Status: item.Status, + FinalStatus: item.FinalStatus, + Message: item.Message, + Reason: item.Reason, + } + res = append(res, e) + } else { + e := &model.EventAndBuild{ + CreateTime: item.CreatedAt.Format(time.DateTime), + TenantID: item.TenantID, + Target: item.Target, + TargetID: item.TargetID, + UserName: item.UserName, + StartTime: item.StartTime, + EndTime: item.EndTime, + OptType: item.OptType, + SynType: string(rune(item.SynType)), + Status: item.Status, + FinalStatus: item.FinalStatus, + Message: item.Message, + Reason: item.Reason, + BuildVersion: version.BuildVersion, + Kind: version.Kind, + DeliveredType: version.DeliveredType, + DeliveredPath: version.DeliveredPath, + ImageName: version.ImageName, + Cmd: version.Cmd, + RepoURL: version.RepoURL, + CodeVersion: version.CodeVersion, + CodeBranch: version.CodeBranch, + CodeCommitMsg: version.CommitMsg, + CodeCommitAuthor: version.Author, + PlanVersion: version.PlanVersion, + } + res = append(res, e) + } + } + return res, nil +} + +// GetLastASyncEvent get last sync event +func (c *EventDaoImpl) GetLastASyncEvent(target, targetID string) (*model.ServiceEvent, error) { + + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "match": { "target": "%s" }}, + { "match": { "target_id": "%s" }}, + { "term": { "syn_type": 0 }} + ] + } + }, + "sort": [ + { "id": "desc" } + ], + "size": 1 +}`, target, targetID) + array, err := c.array(body) + if err != nil || len(array) == 0 { + return nil, err + } + return array[0], nil +} + +// UnfinishedEvents returns unfinished events. +func (c *EventDaoImpl) UnfinishedEvents(target, targetID string, optTypes ...string) ([]*model.ServiceEvent, error) { + op, _ := json.Marshal(optTypes) + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "match": { "target": "%s" }}, + { "match": { "target_id": "%s" }}, + { "match": { "status": "%s" }} + ], + "filter": [ + { "terms": { "opt_type": %s }} + ] + } + } +}`, target, targetID, model.EventStatusFailure.String(), string(op)) + return c.array(body) +} + +// LatestFailurePodEvent returns the latest failure pod event. +func (c *EventDaoImpl) LatestFailurePodEvent(podName string) (*model.ServiceEvent, error) { + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "term": { "target": "%s" } }, + { "term": { "target_id": "%s" } }, + { "term": { "status": "%s" } }, + { "bool": { + "must_not": { "term": { "final_status": "%s" } } + } + } + ] + } + }, + "sort": [ + { "id": { "order": "desc" } } + ], + "size": 1 +}`, model.TargetTypePod, podName, model.EventStatusFailure.String(), model.EventFinalStatusEmptyComplete.String()) + array, err := c.array(body) + if err != nil { + return nil, err + } + return array[0], nil +} + +// GetAppointEvent get event log message +func (c *EventDaoImpl) GetAppointEvent(serviceID, status, Opt string) (*model.ServiceEvent, error) { + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "term": { "service_id": "%s" } }, + { "term": { "status": "%s" } }, + { "term": { "opt_type": "%s" } } + ] + } + }, + "sort": [ + { "id": { "order": "desc" } } + ], + "size": 1 +}`, serviceID, status, Opt) + array, err := c.array(body) + if err != nil { + return nil, err + } + return array[0], nil +} + +// AbnormalEvent Abnormal event in components. +func (c *EventDaoImpl) AbnormalEvent(serviceID, Opt string) (*model.ServiceEvent, error) { + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "term": { "target": "%s" } }, + { "term": { "service_id": "%s" } }, + { "term": { "opt_type": "%s" } }, + { "term": { "status": "%s" } } + ] + } + }, + "sort": [ + { "id": { "order": "desc" } } + ], + "size": 1 +}`, model.TargetTypePod, serviceID, Opt, model.EventStatusFailure.String()) + array, err := c.array(body) + if err != nil { + return nil, err + } + return array[0], nil +} + +// DelAbnormalEvent delete Abnormal event in components. +func (c *EventDaoImpl) DelAbnormalEvent(serviceID, Opt string) error { + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "term": { "target": "%s" } }, + { "term": { "service_id": "%s" } }, + { "term": { "opt_type": "%s" } }, + { "term": { "status": "%s" } } + ] + } + } +}`, model.TargetTypePod, serviceID, Opt, model.EventStatusFailure.String()) + _, err := es.Default().POST("/appstore_tenant_services_event/_delete_by_query", body) + if err != nil { + return err + } + return nil +} + +// DelAllAbnormalEvent delete all Abnormal event in components when stop. +func (c *EventDaoImpl) DelAllAbnormalEvent(serviceID string, Opts []string) error { + optsJson, _ := json.Marshal(Opts) + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "match": { "target": "%s" }}, + { "match": { "service_id": "%s" }}, + { "terms": { "opt_type": %s }}, + { "match": { "status": "%s" }} + ] + } + } +}`, model.TargetTypePod, serviceID, string(optsJson), model.EventStatusFailure.String()) + _, err := es.Default().POST("/appstore_tenant_services_event/_delete_by_query", body) + if err != nil { + return err + } + return nil +} + +// SetEventStatus - +func (c *EventDaoImpl) SetEventStatus(ctx context.Context, status model.EventStatus) error { + event, _ := ctx.Value(ctxutil.ContextKey("event")).(*model.ServiceEvent) + if event != nil { + event.FinalStatus = "complete" + event.Status = string(status) + return c.UpdateModel(event) + } + return nil +} + +// GetExceptionEventsByTime - +func (c *EventDaoImpl) GetExceptionEventsByTime(eventTypes []string, createTime time.Time) ([]*model.ServiceEvent, error) { + eventTypesJson, _ := json.Marshal(eventTypes) + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "terms": { "opt_type": %s } }, + { "range": { "create_time": { "gt": "%s" } } } + ] + } + } +}`, string(eventTypesJson), createTime.Format(time.DateTime)) + return c.array(body) +} + +// CountEvents - +func (c *EventDaoImpl) CountEvents(tenantID, serviceID, eventType string) int64 { + body := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "term": { "tenant_id": "%s" } }, + { "term": { "service_id": "%s" } }, + { "term": { "opt_type": "%s" } } + ] + } + } +}`, tenantID, serviceID, eventType) + post, _ := es.Default().POST("/appstore_tenant_services_event/_count", body) + return gjson.Get(post, "count").Int() +} diff --git a/db/elastic/dao/event_test.go b/db/elastic/dao/event_test.go new file mode 100644 index 0000000000..9ecdea531c --- /dev/null +++ b/db/elastic/dao/event_test.go @@ -0,0 +1,24 @@ +package dao + +import ( + "fmt" + "github.com/tidwall/gjson" + "testing" +) + +func TestJson(t *testing.T) { + fmt.Println(gjson.Get(`{ + "_index": "appstore_tenant_services_event", + "_type": "_doc", + "_id": "1234", + "_version": 4, + "_seq_no": 3, + "_primary_term": 1, + "found": true, + "_source": { + "reason": null, + "event_id": "1234", + "message": "更新的消息1" + } +}`, "_source").Raw) +} diff --git a/db/mysql/dao_impl.go b/db/mysql/dao_impl.go index 5464180e71..4a4248e8c9 100644 --- a/db/mysql/dao_impl.go +++ b/db/mysql/dao_impl.go @@ -19,10 +19,10 @@ package mysql import ( - "github.com/jinzhu/gorm" - "github.com/goodrain/rainbond/db/dao" + elasticdao "github.com/goodrain/rainbond/db/elastic/dao" mysqldao "github.com/goodrain/rainbond/db/mysql/dao" + "github.com/jinzhu/gorm" ) // VolumeTypeDao volumeTypeDao @@ -328,9 +328,7 @@ func (m *Manager) CodeCheckResultDaoTransactions(db *gorm.DB) dao.CodeCheckResul // ServiceEventDao TenantServicePluginRelationDao func (m *Manager) ServiceEventDao() dao.EventDao { - return &mysqldao.EventDaoImpl{ - DB: m.db, - } + return &elasticdao.EventDaoImpl{} } // ServiceEventDaoTransactions TenantServicePluginRelationDaoTransactions diff --git a/pkg/component/core.go b/pkg/component/core.go index 5f92a10a2a..233e62e35d 100644 --- a/pkg/component/core.go +++ b/pkg/component/core.go @@ -29,6 +29,7 @@ import ( "github.com/goodrain/rainbond/mq/mqcomponent/grpcserver" "github.com/goodrain/rainbond/mq/mqcomponent/metrics" "github.com/goodrain/rainbond/mq/mqcomponent/mqclient" + "github.com/goodrain/rainbond/pkg/component/es" "github.com/goodrain/rainbond/pkg/component/grpc" "github.com/goodrain/rainbond/pkg/component/hubregistry" "github.com/goodrain/rainbond/pkg/component/k8s" @@ -140,3 +141,8 @@ func MQGrpcServer() rainbond.ComponentCancel { func MQClient() rainbond.Component { return mqclient.New() } + +// ES - +func ES() rainbond.Component { + return es.New() +} diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go new file mode 100644 index 0000000000..73df12aab1 --- /dev/null +++ b/pkg/component/es/esComponent.go @@ -0,0 +1,88 @@ +package es + +import ( + "bytes" + "context" + "github.com/goodrain/rainbond/config/configs" + "github.com/sirupsen/logrus" + "io" + "net/http" +) + +var defaultEsComponent *Component + +// Component - +type Component struct { + url string + username string + password string +} + +func (c *Component) Start(ctx context.Context, cfg *configs.Config) error { + c.url = cfg.APIConfig.ElasticSearchURL + c.username = cfg.APIConfig.ElasticSearchUsername + c.password = cfg.APIConfig.ElasticSearchPassword + return nil +} + +func (c *Component) CloseHandle() { +} + +// New - +func New() *Component { + defaultEsComponent = &Component{} + return defaultEsComponent +} + +// Default - +func Default() *Component { + return defaultEsComponent +} + +func (c *Component) GET(url string) (string, error) { + return c.request(url, "GET", "") +} + +func (c *Component) POST(url, body string) (string, error) { + return c.request(url, "POST", body) +} + +func (c *Component) PUT(url, body string) (string, error) { + return c.request(url, "PUT", body) +} + +func (c *Component) DELETE(url string) (string, error) { + return c.request(url, "DELETE", "") +} + +func (c *Component) request(url, method, body string) (string, error) { + req, err := http.NewRequest(method, c.url+url, bytes.NewBuffer([]byte(body))) + if err != nil { + logrus.Errorf("Error creating request: %s ", err.Error()) + return "", err + } + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + logrus.Errorf("Error making request: %s", err.Error()) + return "", err + } + defer func() { + if err := resp.Body.Close(); err != nil { + logrus.Errorf("Error closing response body: %s", err.Error()) + } + }() + + if resp.StatusCode < http.StatusOK || resp.StatusCode > 300 { + logrus.Errorf("Error response from server: %d %s\n", resp.StatusCode, resp.Status) + return "", err + } + data, err := io.ReadAll(resp.Body) + if err != nil { + logrus.Errorf("Error reading response body: %s", err.Error()) + return "", err + } + return string(data), nil +} From fe641c26a0d312763dd98eb7844a6d50bdd64058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Thu, 20 Jun 2024 18:28:34 +0800 Subject: [PATCH 02/38] =?UTF-8?q?fix:=20import=20cycle=20not=20allowed=20S?= =?UTF-8?q?igned-off-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888?= =?UTF-8?q?816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/elastic/dao/event_test.go | 24 ------ .../dao/event.go => mysql/dao/event_es.go} | 82 +++++++------------ db/mysql/dao_impl.go | 3 +- 3 files changed, 29 insertions(+), 80 deletions(-) delete mode 100644 db/elastic/dao/event_test.go rename db/{elastic/dao/event.go => mysql/dao/event_es.go} (78%) diff --git a/db/elastic/dao/event_test.go b/db/elastic/dao/event_test.go deleted file mode 100644 index 9ecdea531c..0000000000 --- a/db/elastic/dao/event_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package dao - -import ( - "fmt" - "github.com/tidwall/gjson" - "testing" -) - -func TestJson(t *testing.T) { - fmt.Println(gjson.Get(`{ - "_index": "appstore_tenant_services_event", - "_type": "_doc", - "_id": "1234", - "_version": 4, - "_seq_no": 3, - "_primary_term": 1, - "found": true, - "_source": { - "reason": null, - "event_id": "1234", - "message": "更新的消息1" - } -}`, "_source").Raw) -} diff --git a/db/elastic/dao/event.go b/db/mysql/dao/event_es.go similarity index 78% rename from db/elastic/dao/event.go rename to db/mysql/dao/event_es.go index c1d83f6b9f..bf4ff0275d 100644 --- a/db/elastic/dao/event.go +++ b/db/mysql/dao/event_es.go @@ -13,11 +13,11 @@ import ( "time" ) -type EventDaoImpl struct { +type EventDaoESImpl struct { } // AddModel AddModel -func (c *EventDaoImpl) AddModel(mo model.Interface) error { +func (c *EventDaoESImpl) AddModel(mo model.Interface) error { result := mo.(*model.ServiceEvent) body, _ := json.Marshal(result) _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", result.EventID), string(body)) @@ -28,7 +28,7 @@ func (c *EventDaoImpl) AddModel(mo model.Interface) error { } // UpdateModel UpdateModel -func (c *EventDaoImpl) UpdateModel(mo model.Interface) error { +func (c *EventDaoESImpl) UpdateModel(mo model.Interface) error { update := mo.(*model.ServiceEvent) body, _ := json.Marshal(update) _, err := es.Default().PUT(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", update.EventID), string(body)) @@ -36,13 +36,13 @@ func (c *EventDaoImpl) UpdateModel(mo model.Interface) error { } // DeleteModel DeleteModel -func (c *EventDaoImpl) DeleteModel(id string, args ...interface{}) error { +func (c *EventDaoESImpl) DeleteModel(id string, args ...interface{}) error { _, err := es.Default().DELETE(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", id)) return err } // CreateEventsInBatch creates events in batch. -func (c *EventDaoImpl) CreateEventsInBatch(events []*model.ServiceEvent) error { +func (c *EventDaoESImpl) CreateEventsInBatch(events []*model.ServiceEvent) error { for _, event := range events { _ = c.AddModel(event) } @@ -50,7 +50,7 @@ func (c *EventDaoImpl) CreateEventsInBatch(events []*model.ServiceEvent) error { } // DeleteEvents delete event -func (c *EventDaoImpl) DeleteEvents(eventIDs []string) error { +func (c *EventDaoESImpl) DeleteEvents(eventIDs []string) error { eventIds, _ := json.Marshal(eventIDs) query := fmt.Sprintf(` { @@ -65,7 +65,7 @@ func (c *EventDaoImpl) DeleteEvents(eventIDs []string) error { } // UpdateReason update reasion. -func (c *EventDaoImpl) UpdateReason(eventID string, reason string) error { +func (c *EventDaoESImpl) UpdateReason(eventID string, reason string) error { body := fmt.Sprintf(`{ "script": { "source": "ctx._source.reason = params.reason", @@ -84,7 +84,7 @@ func (c *EventDaoImpl) UpdateReason(eventID string, reason string) error { } // GetEventByEventID get event log message -func (c *EventDaoImpl) GetEventByEventID(eventID string) (*model.ServiceEvent, error) { +func (c *EventDaoESImpl) GetEventByEventID(eventID string) (*model.ServiceEvent, error) { var result model.ServiceEvent get, err := es.Default().GET(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", eventID)) @@ -99,7 +99,7 @@ func (c *EventDaoImpl) GetEventByEventID(eventID string) (*model.ServiceEvent, e } // GetEventByEventIDs get event info -func (c *EventDaoImpl) GetEventByEventIDs(eventIDs []string) ([]*model.ServiceEvent, error) { +func (c *EventDaoESImpl) GetEventByEventIDs(eventIDs []string) ([]*model.ServiceEvent, error) { eventIds, _ := json.Marshal(eventIDs) query := fmt.Sprintf(` { @@ -112,7 +112,7 @@ func (c *EventDaoImpl) GetEventByEventIDs(eventIDs []string) ([]*model.ServiceEv return c.array(query) } -func (c *EventDaoImpl) array(query string) ([]*model.ServiceEvent, error) { +func (c *EventDaoESImpl) array(query string) ([]*model.ServiceEvent, error) { get, err := es.Default().POST("/appstore_tenant_services_event/_search", query) if err != nil { return nil, err @@ -126,7 +126,7 @@ func (c *EventDaoImpl) array(query string) ([]*model.ServiceEvent, error) { } // UpdateInBatch - -func (c *EventDaoImpl) UpdateInBatch(events []*model.ServiceEvent) error { +func (c *EventDaoESImpl) UpdateInBatch(events []*model.ServiceEvent) error { for i := range events { _ = c.UpdateModel(events[i]) } @@ -134,7 +134,7 @@ func (c *EventDaoImpl) UpdateInBatch(events []*model.ServiceEvent) error { } // GetEventByServiceID get event log message -func (c *EventDaoImpl) GetEventByServiceID(serviceID string) ([]*model.ServiceEvent, error) { +func (c *EventDaoESImpl) GetEventByServiceID(serviceID string) ([]*model.ServiceEvent, error) { body := fmt.Sprintf(`{ "query": { "match": { @@ -153,7 +153,7 @@ func (c *EventDaoImpl) GetEventByServiceID(serviceID string) ([]*model.ServiceEv } // DelEventByServiceID delete event log -func (c *EventDaoImpl) DelEventByServiceID(serviceID string) error { +func (c *EventDaoESImpl) DelEventByServiceID(serviceID string) error { query := fmt.Sprintf(` { "query": { @@ -167,7 +167,7 @@ func (c *EventDaoImpl) DelEventByServiceID(serviceID string) error { } // ListByTargetID - -func (c *EventDaoImpl) ListByTargetID(targetID string) ([]*model.ServiceEvent, error) { +func (c *EventDaoESImpl) ListByTargetID(targetID string) ([]*model.ServiceEvent, error) { query := fmt.Sprintf(` { "query": { @@ -180,7 +180,7 @@ func (c *EventDaoImpl) ListByTargetID(targetID string) ([]*model.ServiceEvent, e } // GetEventsByTarget get event by target with page -func (c *EventDaoImpl) GetEventsByTarget(target, targetID string, offset, limit int) ([]*model.ServiceEvent, int, error) { +func (c *EventDaoESImpl) GetEventsByTarget(target, targetID string, offset, limit int) ([]*model.ServiceEvent, int, error) { body := fmt.Sprintf(`{ "query": { "bool": { @@ -209,7 +209,7 @@ func (c *EventDaoImpl) GetEventsByTarget(target, targetID string, offset, limit } // GetEventsByTenantID get event by tenantID -func (c *EventDaoImpl) GetEventsByTenantID(tenantID string, offset, limit int) ([]*model.ServiceEvent, int, error) { +func (c *EventDaoESImpl) GetEventsByTenantID(tenantID string, offset, limit int) ([]*model.ServiceEvent, int, error) { query := fmt.Sprintf(` { "query": { @@ -236,7 +236,7 @@ func (c *EventDaoImpl) GetEventsByTenantID(tenantID string, offset, limit int) ( } // GetEventsByTenantIDs get my teams all event by tenantIDs -func (c *EventDaoImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit int) ([]*model.EventAndBuild, error) { +func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit int) ([]*model.EventAndBuild, error) { tenants, _ := json.Marshal(tenantIDs) body := fmt.Sprintf(`{ "sort": [ @@ -261,39 +261,13 @@ func (c *EventDaoImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit in } } }`, offset, limit, string(tenants)) - - //ops, _ := json.Marshal(tenantIDs) - //// 使用原生 SQL 查询，并进行连接优化 - //query := ` - // SELECT - // a.ID, a.create_time, a.tenant_id, a.target, a.target_id, a.user_name, - // a.start_time, a.end_time, a.opt_type, a.syn_type, a.status, a.final_status, - // a.message, a.reason, b.build_version, b.kind, b.delivered_type, b.delivered_path, - // b.image_name, b.cmd, b.repo_url, b.code_version, b.code_branch, b.code_commit_msg, - // b.code_commit_author, b.plan_version - // FROM - // region.tenant_services_event AS a - // LEFT JOIN - // region.tenant_service_version AS b - // ON - // a.target_id = b.service_id AND a.event_id = b.event_id - // WHERE - // a.target = 'service' - // AND a.tenant_id IN (?) - // ORDER BY - // a.ID DESC - // LIMIT ?, ?; - //` - //if err := c.DB.Debug().Raw(query, tenantIDs, offset, limit).Scan(&events).Error; err != nil { - // return nil, err - //} - array, err := c.array(body) if err != nil { return nil, err } res := make([]*model.EventAndBuild, 0) for _, item := range array { + version, err := db.GetManager().VersionInfoDao().GetVersionByEventID(item.EventID) if err != nil { e := &model.EventAndBuild{ @@ -347,7 +321,7 @@ func (c *EventDaoImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit in } // GetLastASyncEvent get last sync event -func (c *EventDaoImpl) GetLastASyncEvent(target, targetID string) (*model.ServiceEvent, error) { +func (c *EventDaoESImpl) GetLastASyncEvent(target, targetID string) (*model.ServiceEvent, error) { body := fmt.Sprintf(`{ "query": { @@ -372,7 +346,7 @@ func (c *EventDaoImpl) GetLastASyncEvent(target, targetID string) (*model.Servic } // UnfinishedEvents returns unfinished events. -func (c *EventDaoImpl) UnfinishedEvents(target, targetID string, optTypes ...string) ([]*model.ServiceEvent, error) { +func (c *EventDaoESImpl) UnfinishedEvents(target, targetID string, optTypes ...string) ([]*model.ServiceEvent, error) { op, _ := json.Marshal(optTypes) body := fmt.Sprintf(`{ "query": { @@ -392,7 +366,7 @@ func (c *EventDaoImpl) UnfinishedEvents(target, targetID string, optTypes ...str } // LatestFailurePodEvent returns the latest failure pod event. -func (c *EventDaoImpl) LatestFailurePodEvent(podName string) (*model.ServiceEvent, error) { +func (c *EventDaoESImpl) LatestFailurePodEvent(podName string) (*model.ServiceEvent, error) { body := fmt.Sprintf(`{ "query": { "bool": { @@ -420,7 +394,7 @@ func (c *EventDaoImpl) LatestFailurePodEvent(podName string) (*model.ServiceEven } // GetAppointEvent get event log message -func (c *EventDaoImpl) GetAppointEvent(serviceID, status, Opt string) (*model.ServiceEvent, error) { +func (c *EventDaoESImpl) GetAppointEvent(serviceID, status, Opt string) (*model.ServiceEvent, error) { body := fmt.Sprintf(`{ "query": { "bool": { @@ -444,7 +418,7 @@ func (c *EventDaoImpl) GetAppointEvent(serviceID, status, Opt string) (*model.Se } // AbnormalEvent Abnormal event in components. -func (c *EventDaoImpl) AbnormalEvent(serviceID, Opt string) (*model.ServiceEvent, error) { +func (c *EventDaoESImpl) AbnormalEvent(serviceID, Opt string) (*model.ServiceEvent, error) { body := fmt.Sprintf(`{ "query": { "bool": { @@ -469,7 +443,7 @@ func (c *EventDaoImpl) AbnormalEvent(serviceID, Opt string) (*model.ServiceEvent } // DelAbnormalEvent delete Abnormal event in components. -func (c *EventDaoImpl) DelAbnormalEvent(serviceID, Opt string) error { +func (c *EventDaoESImpl) DelAbnormalEvent(serviceID, Opt string) error { body := fmt.Sprintf(`{ "query": { "bool": { @@ -490,7 +464,7 @@ func (c *EventDaoImpl) DelAbnormalEvent(serviceID, Opt string) error { } // DelAllAbnormalEvent delete all Abnormal event in components when stop. -func (c *EventDaoImpl) DelAllAbnormalEvent(serviceID string, Opts []string) error { +func (c *EventDaoESImpl) DelAllAbnormalEvent(serviceID string, Opts []string) error { optsJson, _ := json.Marshal(Opts) body := fmt.Sprintf(`{ "query": { @@ -512,7 +486,7 @@ func (c *EventDaoImpl) DelAllAbnormalEvent(serviceID string, Opts []string) erro } // SetEventStatus - -func (c *EventDaoImpl) SetEventStatus(ctx context.Context, status model.EventStatus) error { +func (c *EventDaoESImpl) SetEventStatus(ctx context.Context, status model.EventStatus) error { event, _ := ctx.Value(ctxutil.ContextKey("event")).(*model.ServiceEvent) if event != nil { event.FinalStatus = "complete" @@ -523,7 +497,7 @@ func (c *EventDaoImpl) SetEventStatus(ctx context.Context, status model.EventSta } // GetExceptionEventsByTime - -func (c *EventDaoImpl) GetExceptionEventsByTime(eventTypes []string, createTime time.Time) ([]*model.ServiceEvent, error) { +func (c *EventDaoESImpl) GetExceptionEventsByTime(eventTypes []string, createTime time.Time) ([]*model.ServiceEvent, error) { eventTypesJson, _ := json.Marshal(eventTypes) body := fmt.Sprintf(`{ "query": { @@ -539,7 +513,7 @@ func (c *EventDaoImpl) GetExceptionEventsByTime(eventTypes []string, createTime } // CountEvents - -func (c *EventDaoImpl) CountEvents(tenantID, serviceID, eventType string) int64 { +func (c *EventDaoESImpl) CountEvents(tenantID, serviceID, eventType string) int64 { body := fmt.Sprintf(`{ "query": { "bool": { diff --git a/db/mysql/dao_impl.go b/db/mysql/dao_impl.go index 4a4248e8c9..dca3084e7b 100644 --- a/db/mysql/dao_impl.go +++ b/db/mysql/dao_impl.go @@ -20,7 +20,6 @@ package mysql import ( "github.com/goodrain/rainbond/db/dao" - elasticdao "github.com/goodrain/rainbond/db/elastic/dao" mysqldao "github.com/goodrain/rainbond/db/mysql/dao" "github.com/jinzhu/gorm" ) @@ -328,7 +327,7 @@ func (m *Manager) CodeCheckResultDaoTransactions(db *gorm.DB) dao.CodeCheckResul // ServiceEventDao TenantServicePluginRelationDao func (m *Manager) ServiceEventDao() dao.EventDao { - return &elasticdao.EventDaoImpl{} + return &mysqldao.EventDaoImpl{} } // ServiceEventDaoTransactions TenantServicePluginRelationDaoTransactions From f9fd975f3ba46f0fd13a3b0e59c1b324eae943ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Thu, 20 Jun 2024 18:33:19 +0800 Subject: [PATCH 03/38] =?UTF-8?q?fix:=20import=20cycle=20not=20allowed=20S?= =?UTF-8?q?igned-off-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888?= =?UTF-8?q?816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 9 ++++++--- db/mysql/dao_impl.go | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index bf4ff0275d..8d3f9c8c99 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -5,15 +5,16 @@ import ( "encoding/json" "fmt" ctxutil "github.com/goodrain/rainbond/api/util/ctx" - "github.com/goodrain/rainbond/db" "github.com/goodrain/rainbond/db/model" "github.com/goodrain/rainbond/pkg/component/es" + "github.com/jinzhu/gorm" "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "time" ) type EventDaoESImpl struct { + DB *gorm.DB } // AddModel AddModel @@ -268,8 +269,8 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit res := make([]*model.EventAndBuild, 0) for _, item := range array { - version, err := db.GetManager().VersionInfoDao().GetVersionByEventID(item.EventID) - if err != nil { + var version model.VersionInfo + if err := c.DB.Where("event_id=?", item.EventID).Find(&version).Error; err != nil { e := &model.EventAndBuild{ CreateTime: item.CreatedAt.Format(time.DateTime), TenantID: item.TenantID, @@ -286,6 +287,7 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit Reason: item.Reason, } res = append(res, e) + } else { e := &model.EventAndBuild{ CreateTime: item.CreatedAt.Format(time.DateTime), @@ -317,6 +319,7 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit res = append(res, e) } } + return res, nil } diff --git a/db/mysql/dao_impl.go b/db/mysql/dao_impl.go index dca3084e7b..4d16f28e3c 100644 --- a/db/mysql/dao_impl.go +++ b/db/mysql/dao_impl.go @@ -327,7 +327,9 @@ func (m *Manager) CodeCheckResultDaoTransactions(db *gorm.DB) dao.CodeCheckResul // ServiceEventDao TenantServicePluginRelationDao func (m *Manager) ServiceEventDao() dao.EventDao { - return &mysqldao.EventDaoImpl{} + return &mysqldao.EventDaoESImpl{ + DB: m.db, + } } // ServiceEventDaoTransactions TenantServicePluginRelationDaoTransactions From 6e6716a921e1a7a540472f43d0d33f824ccd25f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Thu, 20 Jun 2024 18:36:09 +0800 Subject: [PATCH 04/38] =?UTF-8?q?fix:=20undefined:=20time.DateTime=20Signe?= =?UTF-8?q?d-off-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@?= =?UTF-8?q?qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index 8d3f9c8c99..b9ce367581 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -272,7 +272,7 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit var version model.VersionInfo if err := c.DB.Where("event_id=?", item.EventID).Find(&version).Error; err != nil { e := &model.EventAndBuild{ - CreateTime: item.CreatedAt.Format(time.DateTime), + CreateTime: item.CreatedAt.Format("2006-01-02 15:04:05"), TenantID: item.TenantID, Target: item.Target, TargetID: item.TargetID, @@ -290,7 +290,7 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit } else { e := &model.EventAndBuild{ - CreateTime: item.CreatedAt.Format(time.DateTime), + CreateTime: item.CreatedAt.Format("2006-01-02 15:04:05"), TenantID: item.TenantID, Target: item.Target, TargetID: item.TargetID, @@ -511,7 +511,7 @@ func (c *EventDaoESImpl) GetExceptionEventsByTime(eventTypes []string, createTim ] } } -}`, string(eventTypesJson), createTime.Format(time.DateTime)) +}`, string(eventTypesJson), createTime.Format("2006-01-02 15:04:05")) return c.array(body) } From 9f62f86502c184b4bc1a5a3809a763ce30195445 Mon Sep 17 00:00:00 2001 From: DokiDoki1103 <1666888816@qq.com> Date: Sun, 23 Jun 2024 00:25:41 +0800 Subject: [PATCH 05/38] feat: add SetBasicAuth --- pkg/component/es/esComponent.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index 73df12aab1..c970e86086 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -61,6 +61,7 @@ func (c *Component) request(url, method, body string) (string, error) { logrus.Errorf("Error creating request: %s ", err.Error()) return "", err } + req.SetBasicAuth(c.username, c.password) req.Header.Set("Content-Type", "application/json") client := &http.Client{} From c57c3df6a67aa2bc3521cd0787d4e976437b6b7c Mon Sep 17 00:00:00 2001 From: DokiDoki1103 <1666888816@qq.com> Date: Sun, 23 Jun 2024 00:35:08 +0800 Subject: [PATCH 06/38] feat: add SetBasicAuth --- pkg/component/es/esComponent.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index c970e86086..ab894f092a 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -77,6 +77,7 @@ func (c *Component) request(url, method, body string) (string, error) { }() if resp.StatusCode < http.StatusOK || resp.StatusCode > 300 { + logrus.Error(url, body) logrus.Errorf("Error response from server: %d %s\n", resp.StatusCode, resp.Status) return "", err } From ba71c58292964a0a186614361039b96dd0fa8ca2 Mon Sep 17 00:00:00 2001 From: DokiDoki1103 <1666888816@qq.com> Date: Sun, 23 Jun 2024 00:37:33 +0800 Subject: [PATCH 07/38] feat: add SetBasicAuth --- db/mysql/dao/event_es.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index b9ce367581..9ff29bcf93 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -244,7 +244,7 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit { "id": "desc" } ], "from": %d, - "size": %d + "size": %d, "query": { "bool": { "must": [ From 895ffbd99e242cfc9e7599c9110b31906ee8dd5d Mon Sep 17 00:00:00 2001 From: DokiDoki1103 <1666888816@qq.com> Date: Sun, 23 Jun 2024 00:51:43 +0800 Subject: [PATCH 08/38] feat: id sort desc --- db/mysql/dao/event_es.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index 9ff29bcf93..4f47e9a729 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -193,7 +193,7 @@ func (c *EventDaoESImpl) GetEventsByTarget(target, targetID string, offset, limi }, "sort": [ { "create_time": { "order": "desc" } }, - { "id": { "order": "desc" } } + { "ID": { "order": "desc" } } ], "from": %d, "size": %d @@ -220,7 +220,7 @@ func (c *EventDaoESImpl) GetEventsByTenantID(tenantID string, offset, limit int) }, "sort": [ {"start_time": "desc"}, - {"id": "desc"} + {"ID": "desc"} ], "from": %d, "size": %d @@ -241,7 +241,7 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit tenants, _ := json.Marshal(tenantIDs) body := fmt.Sprintf(`{ "sort": [ - { "id": "desc" } + { "ID": "desc" } ], "from": %d, "size": %d, @@ -337,7 +337,7 @@ func (c *EventDaoESImpl) GetLastASyncEvent(target, targetID string) (*model.Serv } }, "sort": [ - { "id": "desc" } + { "ID": "desc" } ], "size": 1 }`, target, targetID) @@ -385,7 +385,7 @@ func (c *EventDaoESImpl) LatestFailurePodEvent(podName string) (*model.ServiceEv } }, "sort": [ - { "id": { "order": "desc" } } + { "ID": { "order": "desc" } } ], "size": 1 }`, model.TargetTypePod, podName, model.EventStatusFailure.String(), model.EventFinalStatusEmptyComplete.String()) @@ -409,7 +409,7 @@ func (c *EventDaoESImpl) GetAppointEvent(serviceID, status, Opt string) (*model. } }, "sort": [ - { "id": { "order": "desc" } } + { "ID": { "order": "desc" } } ], "size": 1 }`, serviceID, status, Opt) @@ -434,7 +434,7 @@ func (c *EventDaoESImpl) AbnormalEvent(serviceID, Opt string) (*model.ServiceEve } }, "sort": [ - { "id": { "order": "desc" } } + { "ID": { "order": "desc" } } ], "size": 1 }`, model.TargetTypePod, serviceID, Opt, model.EventStatusFailure.String()) From 9f07f8543d4de0c00e564063d2e1e84c340b78fd Mon Sep 17 00:00:00 2001 From: DokiDoki1103 <1666888816@qq.com> Date: Sun, 23 Jun 2024 16:15:28 +0800 Subject: [PATCH 09/38] feat: create_time sort desc --- db/mysql/dao/event_es.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index 4f47e9a729..62cc315bdc 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -192,8 +192,7 @@ func (c *EventDaoESImpl) GetEventsByTarget(target, targetID string, offset, limi } }, "sort": [ - { "create_time": { "order": "desc" } }, - { "ID": { "order": "desc" } } + { "create_time": { "order": "desc" } } ], "from": %d, "size": %d @@ -219,8 +218,7 @@ func (c *EventDaoESImpl) GetEventsByTenantID(tenantID string, offset, limit int) } }, "sort": [ - {"start_time": "desc"}, - {"ID": "desc"} + {"start_time": "desc"} ], "from": %d, "size": %d @@ -241,7 +239,7 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit tenants, _ := json.Marshal(tenantIDs) body := fmt.Sprintf(`{ "sort": [ - { "ID": "desc" } + { "create_time": "desc" } ], "from": %d, "size": %d, @@ -337,7 +335,7 @@ func (c *EventDaoESImpl) GetLastASyncEvent(target, targetID string) (*model.Serv } }, "sort": [ - { "ID": "desc" } + { "create_time": "desc" } ], "size": 1 }`, target, targetID) @@ -385,7 +383,7 @@ func (c *EventDaoESImpl) LatestFailurePodEvent(podName string) (*model.ServiceEv } }, "sort": [ - { "ID": { "order": "desc" } } + { "create_time": { "order": "desc" } } ], "size": 1 }`, model.TargetTypePod, podName, model.EventStatusFailure.String(), model.EventFinalStatusEmptyComplete.String()) @@ -409,7 +407,7 @@ func (c *EventDaoESImpl) GetAppointEvent(serviceID, status, Opt string) (*model. } }, "sort": [ - { "ID": { "order": "desc" } } + { "create_time": { "order": "desc" } } ], "size": 1 }`, serviceID, status, Opt) @@ -434,7 +432,7 @@ func (c *EventDaoESImpl) AbnormalEvent(serviceID, Opt string) (*model.ServiceEve } }, "sort": [ - { "ID": { "order": "desc" } } + { "create_time": { "order": "desc" } } ], "size": 1 }`, model.TargetTypePod, serviceID, Opt, model.EventStatusFailure.String()) From 0df2988d63f4aa35a66006e29948ffba7b4bd4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 24 Jun 2024 14:42:38 +0800 Subject: [PATCH 10/38] =?UTF-8?q?fix:=20count=20not=20support=20limit=20Si?= =?UTF-8?q?gned-off-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<16668888?= =?UTF-8?q?16@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index 62cc315bdc..1ffd441b1e 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -203,7 +203,20 @@ func (c *EventDaoESImpl) GetEventsByTarget(target, targetID string, offset, limi if err != nil { return nil, 0, err } - post, _ := es.Default().POST("/appstore_tenant_services_event/_count", body) + + body2 := fmt.Sprintf(`{ + "query": { + "bool": { + "must": [ + { "match": { "target": "%s" } }, + { "match": { "target_id": "%s" } } + ] + } + } +} +`, target, targetID) + + post, _ := es.Default().POST("/appstore_tenant_services_event/_count", body2) return array, (int)(gjson.Get(post, "count").Int()), nil } From 79b8981861ee291e16bb55d222822eb4e6f41bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 24 Jun 2024 15:00:47 +0800 Subject: [PATCH 11/38] =?UTF-8?q?fix:=20count=20not=20support=20limit=20Si?= =?UTF-8?q?gned-off-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<16668888?= =?UTF-8?q?16@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index 1ffd441b1e..ba6fecd191 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -20,6 +20,7 @@ type EventDaoESImpl struct { // AddModel AddModel func (c *EventDaoESImpl) AddModel(mo model.Interface) error { result := mo.(*model.ServiceEvent) + result.CreatedAt = time.Now() body, _ := json.Marshal(result) _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", result.EventID), string(body)) if err != nil { @@ -31,6 +32,7 @@ func (c *EventDaoESImpl) AddModel(mo model.Interface) error { // UpdateModel UpdateModel func (c *EventDaoESImpl) UpdateModel(mo model.Interface) error { update := mo.(*model.ServiceEvent) + update.CreatedAt = time.Now() body, _ := json.Marshal(update) _, err := es.Default().PUT(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", update.EventID), string(body)) return err From 4b42be5b4daac4ad68c0b5c7cbf9285bcc73665f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 24 Jun 2024 15:51:35 +0800 Subject: [PATCH 12/38] =?UTF-8?q?fix:=20time=20format=20error=20Signed-off?= =?UTF-8?q?-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.co?= =?UTF-8?q?m>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/model/event.go | 11 ++++++----- db/mysql/dao/event_es.go | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/db/model/event.go b/db/model/event.go index ccc737ec68..ca4dd7eb57 100644 --- a/db/model/event.go +++ b/db/model/event.go @@ -75,9 +75,10 @@ var EventStatusSuccess EventStatus = "success" // EventStatusFailure - var EventStatusFailure EventStatus = "failure" -//ServiceEvent event struct +// ServiceEvent event struct type ServiceEvent struct { - Model + CreatedAt string `gorm:"column:create_time" json:"create_time"` + EventID string `gorm:"column:event_id;size:40;index:event_id"` TenantID string `gorm:"column:tenant_id;size:40;index:tenant_id"` ServiceID string `gorm:"column:service_id;size:40;index:service_id"` @@ -95,12 +96,12 @@ type ServiceEvent struct { Reason string `gorm:"column:reason"` } -//TableName 表名 +// TableName 表名 func (t *ServiceEvent) TableName() string { return "tenant_services_event" } -//NotificationEvent NotificationEvent +// NotificationEvent NotificationEvent type NotificationEvent struct { Model //Kind could be service, tenant, cluster, node @@ -121,7 +122,7 @@ type NotificationEvent struct { TenantName string `gorm:"column:tenant_name;size:40"` } -//TableName table name +// TableName table name func (n *NotificationEvent) TableName() string { return "region_notification_event" } diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index ba6fecd191..e45854b268 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -20,7 +20,7 @@ type EventDaoESImpl struct { // AddModel AddModel func (c *EventDaoESImpl) AddModel(mo model.Interface) error { result := mo.(*model.ServiceEvent) - result.CreatedAt = time.Now() + result.CreatedAt = time.Now().Format("2006-01-02 15:04:05") body, _ := json.Marshal(result) _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", result.EventID), string(body)) if err != nil { @@ -32,7 +32,7 @@ func (c *EventDaoESImpl) AddModel(mo model.Interface) error { // UpdateModel UpdateModel func (c *EventDaoESImpl) UpdateModel(mo model.Interface) error { update := mo.(*model.ServiceEvent) - update.CreatedAt = time.Now() + update.CreatedAt = time.Now().Format("2006-01-02 15:04:05") body, _ := json.Marshal(update) _, err := es.Default().PUT(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", update.EventID), string(body)) return err @@ -285,7 +285,7 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit var version model.VersionInfo if err := c.DB.Where("event_id=?", item.EventID).Find(&version).Error; err != nil { e := &model.EventAndBuild{ - CreateTime: item.CreatedAt.Format("2006-01-02 15:04:05"), + CreateTime: item.CreatedAt, TenantID: item.TenantID, Target: item.Target, TargetID: item.TargetID, @@ -303,7 +303,7 @@ func (c *EventDaoESImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit } else { e := &model.EventAndBuild{ - CreateTime: item.CreatedAt.Format("2006-01-02 15:04:05"), + CreateTime: item.CreatedAt, TenantID: item.TenantID, Target: item.Target, TargetID: item.TargetID, From 7f5de6389aab109005fb59faa52f1c3901feef9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 24 Jun 2024 16:04:04 +0800 Subject: [PATCH 13/38] =?UTF-8?q?fix:=20time=20format=20error=20Signed-off?= =?UTF-8?q?-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.co?= =?UTF-8?q?m>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/model/event.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/model/event.go b/db/model/event.go index ca4dd7eb57..68fa977cef 100644 --- a/db/model/event.go +++ b/db/model/event.go @@ -77,8 +77,8 @@ var EventStatusFailure EventStatus = "failure" // ServiceEvent event struct type ServiceEvent struct { - CreatedAt string `gorm:"column:create_time" json:"create_time"` - + ID uint `gorm:"column:ID;primary_key"` + CreatedAt string `gorm:"column:create_time" json:"create_time"` EventID string `gorm:"column:event_id;size:40;index:event_id"` TenantID string `gorm:"column:tenant_id;size:40;index:tenant_id"` ServiceID string `gorm:"column:service_id;size:40;index:service_id"` From 61853c805e8a492dc6c898200f3ef82bcfab8cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 24 Jun 2024 16:52:16 +0800 Subject: [PATCH 14/38] =?UTF-8?q?fix:=20time=20format=20error=20Signed-off?= =?UTF-8?q?-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.co?= =?UTF-8?q?m>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/component/es/esComponent.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index ab894f092a..2d768214fb 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -86,5 +86,7 @@ func (c *Component) request(url, method, body string) (string, error) { logrus.Errorf("Error reading response body: %s", err.Error()) return "", err } + logrus.Infof("url: %s", url) + logrus.Infof("body: %s", body) return string(data), nil } From 6989a7f30b85b1257e7bf0c87c1eaef6af41c006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 24 Jun 2024 17:16:24 +0800 Subject: [PATCH 15/38] =?UTF-8?q?fix:=20field=20Signed-off-by:=20=E9=80=86?= =?UTF-8?q?=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/model/event.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/db/model/event.go b/db/model/event.go index 68fa977cef..333cb50c45 100644 --- a/db/model/event.go +++ b/db/model/event.go @@ -77,23 +77,23 @@ var EventStatusFailure EventStatus = "failure" // ServiceEvent event struct type ServiceEvent struct { - ID uint `gorm:"column:ID;primary_key"` + ID uint `gorm:"column:ID;primary_key" json:"-"` CreatedAt string `gorm:"column:create_time" json:"create_time"` - EventID string `gorm:"column:event_id;size:40;index:event_id"` - TenantID string `gorm:"column:tenant_id;size:40;index:tenant_id"` - ServiceID string `gorm:"column:service_id;size:40;index:service_id"` - Target string `gorm:"column:target;size:40"` - TargetID string `gorm:"column:target_id;size:255;index:target_id"` - RequestBody string `gorm:"column:request_body;size:1024"` - UserName string `gorm:"column:user_name;size:40"` - StartTime string `gorm:"column:start_time;size:40"` - EndTime string `gorm:"column:end_time;size:40"` - OptType string `gorm:"column:opt_type;size:40"` - SynType int `gorm:"column:syn_type;size:1"` - Status string `gorm:"column:status;size:40"` - FinalStatus string `gorm:"column:final_status;size:40"` - Message string `gorm:"column:message"` - Reason string `gorm:"column:reason"` + EventID string `gorm:"column:event_id;size:40;index:event_id" json:"event_id"` + TenantID string `gorm:"column:tenant_id;size:40;index:tenant_id" json:"tenant_id"` + ServiceID string `gorm:"column:service_id;size:40;index:service_id" json:"service_id"` + Target string `gorm:"column:target;size:40" json:"target"` + TargetID string `gorm:"column:target_id;size:255;index:target_id" json:"target_id"` + RequestBody string `gorm:"column:request_body;size:1024" json:"request_body"` + UserName string `gorm:"column:user_name;size:40" json:"user_name"` + StartTime string `gorm:"column:start_time;size:40" json:"start_time"` + EndTime string `gorm:"column:end_time;size:40" json:"end_time"` + OptType string `gorm:"column:opt_type;size:40" json:"opt_type"` + SynType int `gorm:"column:syn_type;size:1" json:"syn_type"` + Status string `gorm:"column:status;size:40" json:"status"` + FinalStatus string `gorm:"column:final_status;size:40" json:"final_status"` + Message string `gorm:"column:message" json:"message"` + Reason string `gorm:"column:reason" json:"reason"` } // TableName 表名 From 21f4862d1f6ad6098b605743e6c58a63a6854377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 24 Jun 2024 17:36:16 +0800 Subject: [PATCH 16/38] =?UTF-8?q?fix:=20format=20date=20Signed-off-by:=20?= =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/handler/service_batch_operation.go | 16 ++++++++-------- builder/exector/exector.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/handler/service_batch_operation.go b/api/handler/service_batch_operation.go index 53afeca014..db0c663d96 100644 --- a/api/handler/service_batch_operation.go +++ b/api/handler/service_batch_operation.go @@ -38,7 +38,7 @@ import ( "github.com/sirupsen/logrus" ) -//BatchOperationHandler batch operation handler +// BatchOperationHandler batch operation handler type BatchOperationHandler struct { mqCli gclient.MQClient operationHandler *OperationHandler @@ -48,12 +48,12 @@ type BatchOperationHandler struct { EventIDs []string } -//BatchOperationResult batch operation result +// BatchOperationResult batch operation result type BatchOperationResult struct { BatchResult []OperationResult `json:"batche_result"` } -//CreateBatchOperationHandler create batch operation handler +// CreateBatchOperationHandler create batch operation handler func CreateBatchOperationHandler(mqCli gclient.MQClient, statusCli *client.AppRuntimeSyncClient, operationHandler *OperationHandler) *BatchOperationHandler { return &BatchOperationHandler{ mqCli: mqCli, @@ -72,7 +72,7 @@ func (b *BatchOperationHandler) serviceStartupSequence(serviceIDs []string) map[ return startupSeqConfigs } -//Build build +// Build build func (b *BatchOperationHandler) Build(ctx context.Context, tenant *dbmodel.Tenants, operator string, batchOpReqs model.BatchOpRequesters) (model.BatchOpResult, error) { if logrus.IsLevelEnabled(logrus.DebugLevel) { defer util.Elapsed("[BatchOperationHandler] build components")() @@ -118,7 +118,7 @@ func (b *BatchOperationHandler) Build(ctx context.Context, tenant *dbmodel.Tenan return batchOpResult, nil } -//Start batch start +// Start batch start func (b *BatchOperationHandler) Start(ctx context.Context, tenant *dbmodel.Tenants, operator string, batchOpReqs model.BatchOpRequesters) (model.BatchOpResult, error) { if logrus.IsLevelEnabled(logrus.DebugLevel) { defer util.Elapsed("[BatchOperationHandler] start components")() @@ -165,7 +165,7 @@ func (b *BatchOperationHandler) Start(ctx context.Context, tenant *dbmodel.Tenan return batchOpResult, nil } -//Stop batch stop +// Stop batch stop func (b *BatchOperationHandler) Stop(ctx context.Context, tenant *dbmodel.Tenants, operator string, batchOpReqs model.BatchOpRequesters) (model.BatchOpResult, error) { if logrus.IsLevelEnabled(logrus.DebugLevel) { defer util.Elapsed("[BatchOperationHandler] stop components")() @@ -197,7 +197,7 @@ func (b *BatchOperationHandler) Stop(ctx context.Context, tenant *dbmodel.Tenant return batchOpResult, nil } -//Upgrade batch upgrade +// Upgrade batch upgrade func (b *BatchOperationHandler) Upgrade(ctx context.Context, tenant *dbmodel.Tenants, operator string, batchOpReqs model.BatchOpRequesters) (model.BatchOpResult, error) { if logrus.IsLevelEnabled(logrus.DebugLevel) { defer util.Elapsed("[BatchOperationHandler] upgrade components")() @@ -280,7 +280,7 @@ func (b *BatchOperationHandler) createEvents(tenantID, operator string, batchOpR Target: dbmodel.TargetTypeService, TargetID: req.GetComponentID(), UserName: operator, - StartTime: time.Now().Format(time.RFC3339), + StartTime: time.Now().Format("2006-01-02 15:04:05"), SynType: dbmodel.ASYNEVENTTYPE, OptType: req.OpType(), } diff --git a/builder/exector/exector.go b/builder/exector/exector.go index c149f897ff..e544d6cab4 100644 --- a/builder/exector/exector.go +++ b/builder/exector/exector.go @@ -509,7 +509,7 @@ func (e *exectorManager) sendAction(tenantID, serviceID, eventID, newVersion, ac EventID: util.NewUUID(), TenantID: tenantID, ServiceID: serviceID, - StartTime: time.Now().Format(time.RFC3339), + StartTime: time.Now().Format("2006-01-02 15:04:05"), OptType: "upgrade", Target: "service", TargetID: serviceID, From bae930d2b327ad0020092bac6e8b85ca0638bd6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 24 Jun 2024 17:38:38 +0800 Subject: [PATCH 17/38] =?UTF-8?q?fix:=20format=20date=20Signed-off-by:=20?= =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index e45854b268..dfab6d046d 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -21,6 +21,13 @@ type EventDaoESImpl struct { func (c *EventDaoESImpl) AddModel(mo model.Interface) error { result := mo.(*model.ServiceEvent) result.CreatedAt = time.Now().Format("2006-01-02 15:04:05") + if result.StartTime == "" { + result.StartTime = time.Now().Format("2006-01-02 15:04:05") + } + + if result.EndTime == "" { + result.EndTime = result.StartTime + } body, _ := json.Marshal(result) _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", result.EventID), string(body)) if err != nil { From d63d4bd9d42b5ad5fe036ffa82b19bbae61e0f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 24 Jun 2024 18:25:28 +0800 Subject: [PATCH 18/38] =?UTF-8?q?fix:=20format=20date=20Signed-off-by:=20?= =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index dfab6d046d..8181e3032f 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -21,13 +21,8 @@ type EventDaoESImpl struct { func (c *EventDaoESImpl) AddModel(mo model.Interface) error { result := mo.(*model.ServiceEvent) result.CreatedAt = time.Now().Format("2006-01-02 15:04:05") - if result.StartTime == "" { - result.StartTime = time.Now().Format("2006-01-02 15:04:05") - } - - if result.EndTime == "" { - result.EndTime = result.StartTime - } + result.StartTime = time.Now().Format("2006-01-02 15:04:05") + result.EndTime = result.StartTime body, _ := json.Marshal(result) _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", result.EventID), string(body)) if err != nil { From 3a734947c75459c7db23daa1c7c602922a986664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Tue, 25 Jun 2024 10:37:53 +0800 Subject: [PATCH 19/38] =?UTF-8?q?fix:=20format=20array=20Signed-off-by:=20?= =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/api/option/option.go | 2 +- db/mysql/dao/event_es.go | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/api/option/option.go b/cmd/api/option/option.go index de3c8918b4..9b2c1795d4 100644 --- a/cmd/api/option/option.go +++ b/cmd/api/option/option.go @@ -127,7 +127,7 @@ func (a *APIServer) AddFlags(fs *pflag.FlagSet) { fs.StringSliceVar(&a.EventLogServers, "event-servers", []string{"rbd-eventlog:6366"}, "event log server address") fs.StringSliceVar(&a.EventLogEndpoints, "event-log", []string{"local=>rbd-eventlog:6363"}, "event log websocket address") - fs.StringVar(&a.ElasticSearchURL, "es-url", "127.0.0.1:9200", "es url") + fs.StringVar(&a.ElasticSearchURL, "es-url", "http://47.92.106.114:9200", "es url") fs.StringVar(&a.ElasticSearchUsername, "es-username", "", "es username") fs.StringVar(&a.ElasticSearchPassword, "es-password", "", "es pwd") } diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index 8181e3032f..c00033d4f1 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -123,9 +123,11 @@ func (c *EventDaoESImpl) array(query string) ([]*model.ServiceEvent, error) { return nil, err } var result []*model.ServiceEvent - err = json.Unmarshal([]byte(gjson.Get(get, "hits.hits").Raw), &result) - if err != nil { - return nil, err + fmt.Println(gjson.Get(get, "hits.hits").Raw) + for _, hit := range gjson.Get(get, "hits.hits").Array() { + var s *model.ServiceEvent + _ = json.Unmarshal([]byte(hit.Get("_source").Raw), &s) + result = append(result, s) } return result, nil } From 44c254ff7003057a6019ea57898912dcb14ef637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Tue, 25 Jun 2024 11:48:20 +0800 Subject: [PATCH 20/38] =?UTF-8?q?fix:=20format=20array=20Signed-off-by:=20?= =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index c00033d4f1..f4b2fe292f 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -35,8 +35,10 @@ func (c *EventDaoESImpl) AddModel(mo model.Interface) error { func (c *EventDaoESImpl) UpdateModel(mo model.Interface) error { update := mo.(*model.ServiceEvent) update.CreatedAt = time.Now().Format("2006-01-02 15:04:05") + update.StartTime = time.Now().Format("2006-01-02 15:04:05") + update.EndTime = update.StartTime body, _ := json.Marshal(update) - _, err := es.Default().PUT(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", update.EventID), string(body)) + _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", update.EventID), string(body)) return err } From 6ee92d6ba3f1222fa98214382eb702e488726d3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Tue, 25 Jun 2024 14:09:47 +0800 Subject: [PATCH 21/38] feat: add es --- cmd/builder/option/option.go | 8 ++++++++ cmd/builder/server/server.go | 3 +++ cmd/eventlog/server/server.go | 7 +++++++ cmd/worker/option/option.go | 8 ++++++++ cmd/worker/server/server.go | 4 ++++ eventlog/conf/conf.go | 4 ++++ pkg/component/es/esComponent.go | 6 ++++++ 7 files changed, 40 insertions(+) diff --git a/cmd/builder/option/option.go b/cmd/builder/option/option.go index daa9cc050e..d16b262e58 100644 --- a/cmd/builder/option/option.go +++ b/cmd/builder/option/option.go @@ -59,6 +59,10 @@ type Config struct { KeepCount int CleanInterval int BRVersion string + + ElasticSearchURL string + ElasticSearchUsername string + ElasticSearchPassword string } // Builder builder server @@ -106,6 +110,10 @@ func (a *Builder) AddFlags(fs *pflag.FlagSet) { fs.StringSliceVar(&a.EventLogServers, "event-servers", []string{"rbd-eventlog:6366"}, "event log server address. simple lb") fs.StringVar(&a.MQAPI, "mq-api", "rbd-mq:6300", "acp_mq api") + fs.StringVar(&a.ElasticSearchURL, "es-url", "http://47.92.106.114:9200", "es url") + fs.StringVar(&a.ElasticSearchUsername, "es-username", "", "es username") + fs.StringVar(&a.ElasticSearchPassword, "es-password", "", "es pwd") + } // SetLog 设置log diff --git a/cmd/builder/server/server.go b/cmd/builder/server/server.go index 648d4fe673..8425982db5 100644 --- a/cmd/builder/server/server.go +++ b/cmd/builder/server/server.go @@ -30,6 +30,7 @@ import ( "github.com/goodrain/rainbond/db/config" "github.com/goodrain/rainbond/event" "github.com/goodrain/rainbond/mq/client" + "github.com/goodrain/rainbond/pkg/component/es" "github.com/goodrain/rainbond/pkg/gogo" k8sutil "github.com/goodrain/rainbond/util/k8s" "github.com/prometheus/client_golang/prometheus" @@ -51,6 +52,8 @@ func Run(s *option.Builder) error { MysqlConnectionInfo: s.Config.MysqlConnectionInfo, } + es.New().SingleStart(s.Config.ElasticSearchURL, s.Config.ElasticSearchUsername, s.Config.ElasticSearchPassword) + if err := db.CreateManager(dbconfig); err != nil { return err } diff --git a/cmd/eventlog/server/server.go b/cmd/eventlog/server/server.go index 12061c0a7d..1cfc2f656b 100644 --- a/cmd/eventlog/server/server.go +++ b/cmd/eventlog/server/server.go @@ -20,6 +20,7 @@ package server import ( "fmt" + "github.com/goodrain/rainbond/pkg/component/es" "os" "os/signal" "path" @@ -101,6 +102,10 @@ func (s *LogServer) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&s.Conf.Entry.NewMonitorMessageServerConf.ListenerPort, "monitor.udp.port", 6166, "receive new monitor udp server port") fs.StringVar(&s.Conf.Cluster.Discover.NodeID, "node-id", "", "the unique ID for this node.") fs.DurationVar(&s.Conf.Cluster.PubSub.PollingTimeout, "zmq4-polling-timeout", 200*time.Millisecond, "The timeout determines the time-out on the polling of sockets") + + fs.StringVar(&s.Conf.ElasticSearchURL, "es-url", "http://47.92.106.114:9200", "es url") + fs.StringVar(&s.Conf.ElasticSearchUsername, "es-username", "", "es username") + fs.StringVar(&s.Conf.ElasticSearchPassword, "es-password", "", "es pwd") } // InitLog 初始化log @@ -161,6 +166,8 @@ func (s *LogServer) Run() error { s.Logger.Debug("Start run server.") log := s.Logger + es.New().SingleStart(s.Conf.ElasticSearchURL, s.Conf.ElasticSearchUsername, s.Conf.ElasticSearchPassword) + //init new db if err := db.CreateDBManager(s.Conf.EventStore.DB); err != nil { logrus.Infof("create db manager error, %v", err) diff --git a/cmd/worker/option/option.go b/cmd/worker/option/option.go index ccb72b1913..763041178f 100644 --- a/cmd/worker/option/option.go +++ b/cmd/worker/option/option.go @@ -53,6 +53,10 @@ type Config struct { RBDNamespace string GrdataPVCName string Helm Helm + + ElasticSearchURL string + ElasticSearchUsername string + ElasticSearchPassword string } // Helm helm configuration. @@ -104,6 +108,10 @@ func (a *Worker) AddFlags(fs *pflag.FlagSet) { a.Helm.RepoFile = path.Join(a.Helm.DataDir, "repo/repositories.yaml") a.Helm.RepoCache = path.Join(a.Helm.DataDir, "cache") a.Helm.ChartCache = path.Join(a.Helm.DataDir, "chart") + + fs.StringVar(&a.ElasticSearchURL, "es-url", "http://47.92.106.114:9200", "es url") + fs.StringVar(&a.ElasticSearchUsername, "es-username", "", "es username") + fs.StringVar(&a.ElasticSearchPassword, "es-password", "", "es pwd") } // SetLog 设置log diff --git a/cmd/worker/server/server.go b/cmd/worker/server/server.go index 8b62d0009c..c32836eb09 100644 --- a/cmd/worker/server/server.go +++ b/cmd/worker/server/server.go @@ -21,6 +21,7 @@ package server import ( apisixversioned "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/client/clientset/versioned" "github.com/goodrain/rainbond/otherclient" + "github.com/goodrain/rainbond/pkg/component/es" "k8s.io/client-go/restmapper" "kubevirt.io/client-go/kubecli" "os" @@ -56,6 +57,9 @@ func Run(s *option.Worker) error { DBType: s.Config.DBType, MysqlConnectionInfo: s.Config.MysqlConnectionInfo, } + + es.New().SingleStart(s.Config.ElasticSearchURL, s.Config.ElasticSearchUsername, s.Config.ElasticSearchPassword) + //step 1:db manager init ,event log client init if err := db.CreateManager(dbconfig); err != nil { return err diff --git a/eventlog/conf/conf.go b/eventlog/conf/conf.go index 2f842c9d2c..02511e2e56 100644 --- a/eventlog/conf/conf.go +++ b/eventlog/conf/conf.go @@ -29,6 +29,10 @@ type Conf struct { WebHook WebHookConf Cluster ClusterConf Kubernetes KubernetsConf + + ElasticSearchURL string + ElasticSearchUsername string + ElasticSearchPassword string } // WebHookConf webhook conf diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index 2d768214fb..09d9f2143b 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -25,6 +25,12 @@ func (c *Component) Start(ctx context.Context, cfg *configs.Config) error { return nil } +func (c *Component) SingleStart(url, username, password string) { + c.url = url + c.username = username + c.password = password +} + func (c *Component) CloseHandle() { } From 80615cac73d7bfa8ddf8044c5c0c1918dad82480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Tue, 25 Jun 2024 14:31:44 +0800 Subject: [PATCH 22/38] =?UTF-8?q?fix:=20all=20link=20es=20Signed-off-by:?= =?UTF-8?q?=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index f4b2fe292f..eff3a929d5 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -409,7 +409,7 @@ func (c *EventDaoESImpl) LatestFailurePodEvent(podName string) (*model.ServiceEv "size": 1 }`, model.TargetTypePod, podName, model.EventStatusFailure.String(), model.EventFinalStatusEmptyComplete.String()) array, err := c.array(body) - if err != nil { + if err != nil || len(array) == 0 { return nil, err } return array[0], nil @@ -433,7 +433,7 @@ func (c *EventDaoESImpl) GetAppointEvent(serviceID, status, Opt string) (*model. "size": 1 }`, serviceID, status, Opt) array, err := c.array(body) - if err != nil { + if err != nil || len(array) == 0 { return nil, err } return array[0], nil @@ -458,7 +458,7 @@ func (c *EventDaoESImpl) AbnormalEvent(serviceID, Opt string) (*model.ServiceEve "size": 1 }`, model.TargetTypePod, serviceID, Opt, model.EventStatusFailure.String()) array, err := c.array(body) - if err != nil { + if err != nil || len(array) == 0 { return nil, err } return array[0], nil From fa6b7d9e28546d81cc603babfad1dd0f64596cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=90=AF=E8=88=AA?= <101104760+ZhangSetSail@users.noreply.github.com> Date: Wed, 26 Jun 2024 17:23:53 +0800 Subject: [PATCH 23/38] feat: add es --- db/mysql/dao/event.go | 71 +++++++++++++++++++-------------- pkg/component/es/esComponent.go | 7 +++- 2 files changed, 47 insertions(+), 31 deletions(-) diff --git a/db/mysql/dao/event.go b/db/mysql/dao/event.go index db8b38a597..b8b311ab0c 100644 --- a/db/mysql/dao/event.go +++ b/db/mysql/dao/event.go @@ -30,7 +30,7 @@ import ( "time" ) -//AddModel AddModel +// AddModel AddModel func (c *EventDaoImpl) AddModel(mo model.Interface) error { result := mo.(*model.ServiceEvent) var oldResult model.ServiceEvent @@ -45,7 +45,7 @@ func (c *EventDaoImpl) AddModel(mo model.Interface) error { return nil } -//UpdateModel UpdateModel +// UpdateModel UpdateModel func (c *EventDaoImpl) UpdateModel(mo model.Interface) error { update := mo.(*model.ServiceEvent) var oldResult model.ServiceEvent @@ -58,7 +58,7 @@ func (c *EventDaoImpl) UpdateModel(mo model.Interface) error { return nil } -//EventDaoImpl EventLogMessageDaoImpl +// EventDaoImpl EventLogMessageDaoImpl type EventDaoImpl struct { DB *gorm.DB } @@ -87,7 +87,7 @@ func (c *EventDaoImpl) CreateEventsInBatch(events []*model.ServiceEvent) error { return nil } -//DeleteEvents delete event +// DeleteEvents delete event func (c *EventDaoImpl) DeleteEvents(eventIDs []string) error { return c.DB.Where("event_id in (?)", eventIDs).Delete(&model.ServiceEvent{}).Error } @@ -97,7 +97,7 @@ func (c *EventDaoImpl) UpdateReason(eventID string, reason string) error { return c.DB.Model(&model.ServiceEvent{}).Where("event_id=?", eventID).UpdateColumn("reason", reason).Error } -//GetEventByEventID get event log message +// GetEventByEventID get event log message func (c *EventDaoImpl) GetEventByEventID(eventID string) (*model.ServiceEvent, error) { var result model.ServiceEvent if err := c.DB.Where("event_id=?", eventID).Find(&result).Error; err != nil { @@ -106,7 +106,7 @@ func (c *EventDaoImpl) GetEventByEventID(eventID string) (*model.ServiceEvent, e return &result, nil } -//GetEventByEventIDs get event info +// GetEventByEventIDs get event info func (c *EventDaoImpl) GetEventByEventIDs(eventIDs []string) ([]*model.ServiceEvent, error) { var result []*model.ServiceEvent if err := c.DB.Where("event_id in (?)", eventIDs).Find(&result).Error; err != nil { @@ -141,7 +141,7 @@ func (c *EventDaoImpl) UpdateInBatch(events []*model.ServiceEvent) error { return nil } -//GetEventByServiceID get event log message +// GetEventByServiceID get event log message func (c *EventDaoImpl) GetEventByServiceID(serviceID string) ([]*model.ServiceEvent, error) { var result []*model.ServiceEvent if err := c.DB.Where("service_id=?", serviceID).Find(&result).Order("start_time DESC").Error; err != nil { @@ -153,7 +153,7 @@ func (c *EventDaoImpl) GetEventByServiceID(serviceID string) ([]*model.ServiceEv return result, nil } -//DelEventByServiceID delete event log +// DelEventByServiceID delete event log func (c *EventDaoImpl) DelEventByServiceID(serviceID string) error { var result []*model.ServiceEvent isNoteExist := c.DB.Where("service_id=?", serviceID).Find(&result).RecordNotFound() @@ -220,23 +220,36 @@ func (c *EventDaoImpl) GetEventsByTenantID(tenantID string, offset, limit int) ( // GetEventsByTenantIDs get my teams all event by tenantIDs func (c *EventDaoImpl) GetEventsByTenantIDs(tenantIDs []string, offset, limit int) ([]*model.EventAndBuild, error) { - var EventAndBuild []*model.EventAndBuild - if err := c.DB.Raw("select a.create_time, a.tenant_id, a.target, a.target_id, a.user_name, a.start_time, a.end_time, a.opt_type, a.syn_type, a.status, a.final_status, a.message, a.reason "+ - "build_version, kind, delivered_type, delivered_path, image_name, cmd, repo_url, code_version, code_branch, code_commit_msg, code_commit_author, plan_version "+ - "from "+ - "(select create_time, tenant_id, target, target_id, user_name, start_time, end_time, opt_type, syn_type, status, final_status, message, reason, event_id "+ - "from tenant_services_event "+ - "where target = 'service' "+ - "and tenant_id in (?)) as a "+ - "left join "+ - "tenant_service_version "+ - "on a.target_id = tenant_service_version.service_id and a.event_id = tenant_service_version.event_id", tenantIDs).Order("start_time DESC").Offset(offset).Limit(limit).Scan(&EventAndBuild).Error; err != nil { + var events []*model.EventAndBuild + + // 使用原生 SQL 查询，并进行连接优化 + query := ` + SELECT + a.ID, a.create_time, a.tenant_id, a.target, a.target_id, a.user_name, + a.start_time, a.end_time, a.opt_type, a.syn_type, a.status, a.final_status, + a.message, a.reason, b.build_version, b.kind, b.delivered_type, b.delivered_path, + b.image_name, b.cmd, b.repo_url, b.code_version, b.code_branch, b.code_commit_msg, + b.code_commit_author, b.plan_version + FROM + tenant_services_event AS a + LEFT JOIN + tenant_service_version AS b + ON + a.target_id = b.service_id AND a.event_id = b.event_id + WHERE + a.target = 'service' + AND a.tenant_id IN (?) + ORDER BY + a.ID DESC + LIMIT ?, ?; + ` + if err := c.DB.Debug().Raw(query, tenantIDs, offset, limit).Scan(&events).Error; err != nil { return nil, err } - return EventAndBuild, nil + return events, nil } -//GetLastASyncEvent get last sync event +// GetLastASyncEvent get last sync event func (c *EventDaoImpl) GetLastASyncEvent(target, targetID string) (*model.ServiceEvent, error) { var result model.ServiceEvent if err := c.DB.Where("target=? and target_id=? and syn_type=0", target, targetID).Last(&result).Error; err != nil { @@ -265,7 +278,7 @@ func (c *EventDaoImpl) LatestFailurePodEvent(podName string) (*model.ServiceEven return &event, nil } -//GetAppointEvent get event log message +// GetAppointEvent get event log message func (c *EventDaoImpl) GetAppointEvent(serviceID, status, Opt string) (*model.ServiceEvent, error) { var result model.ServiceEvent if err := c.DB.Where("service_id=? and status=? and opt_type=?", serviceID, status, Opt).Last(&result).Error; err != nil { @@ -315,12 +328,12 @@ func (c *EventDaoImpl) SetEventStatus(ctx context.Context, status model.EventSta return nil } -//NotificationEventDaoImpl NotificationEventDaoImpl +// NotificationEventDaoImpl NotificationEventDaoImpl type NotificationEventDaoImpl struct { DB *gorm.DB } -//AddModel AddModel +// AddModel AddModel func (c *NotificationEventDaoImpl) AddModel(mo model.Interface) error { result := mo.(*model.NotificationEvent) result.LastTime = time.Now() @@ -337,7 +350,7 @@ func (c *NotificationEventDaoImpl) AddModel(mo model.Interface) error { return nil } -//UpdateModel UpdateModel +// UpdateModel UpdateModel func (c *NotificationEventDaoImpl) UpdateModel(mo model.Interface) error { result := mo.(*model.NotificationEvent) var oldResult model.NotificationEvent @@ -349,7 +362,7 @@ func (c *NotificationEventDaoImpl) UpdateModel(mo model.Interface) error { return gorm.ErrRecordNotFound } -//GetNotificationEventByKind GetNotificationEventByKind +// GetNotificationEventByKind GetNotificationEventByKind func (c *NotificationEventDaoImpl) GetNotificationEventByKind(kind, kindID string) ([]*model.NotificationEvent, error) { var result []*model.NotificationEvent if err := c.DB.Where("kind=? and kind_id=?", kind, kindID).Find(&result).Order("last_time DESC").Error; err != nil { @@ -361,7 +374,7 @@ func (c *NotificationEventDaoImpl) GetNotificationEventByKind(kind, kindID strin return result, nil } -//GetNotificationEventByTime GetNotificationEventByTime +// GetNotificationEventByTime GetNotificationEventByTime func (c *NotificationEventDaoImpl) GetNotificationEventByTime(start, end time.Time) ([]*model.NotificationEvent, error) { var result []*model.NotificationEvent if !start.IsZero() && !end.IsZero() { @@ -382,7 +395,7 @@ func (c *NotificationEventDaoImpl) GetNotificationEventByTime(start, end time.Ti return result, nil } -//GetNotificationEventNotHandle GetNotificationEventNotHandle +// GetNotificationEventNotHandle GetNotificationEventNotHandle func (c *NotificationEventDaoImpl) GetNotificationEventNotHandle() ([]*model.NotificationEvent, error) { var result []*model.NotificationEvent if err := c.DB.Where("is_handle=?", false).Find(&result).Order("last_time DESC").Error; err != nil { @@ -394,7 +407,7 @@ func (c *NotificationEventDaoImpl) GetNotificationEventNotHandle() ([]*model.Not return result, nil } -//GetNotificationEventByHash GetNotificationEventByHash +// GetNotificationEventByHash GetNotificationEventByHash func (c *NotificationEventDaoImpl) GetNotificationEventByHash(hash string) (*model.NotificationEvent, error) { var result model.NotificationEvent if err := c.DB.Where("hash=?", hash).Find(&result).Error; err != nil { diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index 09d9f2143b..4e3deb1716 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -3,6 +3,7 @@ package es import ( "bytes" "context" + "crypto/tls" "github.com/goodrain/rainbond/config/configs" "github.com/sirupsen/logrus" "io" @@ -69,8 +70,10 @@ func (c *Component) request(url, method, body string) (string, error) { } req.SetBasicAuth(c.username, c.password) req.Header.Set("Content-Type", "application/json") - - client := &http.Client{} + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} resp, err := client.Do(req) if err != nil { logrus.Errorf("Error making request: %s", err.Error()) From 28a9ee0bece26e119d240381c258b2535b661b67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=90=AF=E8=88=AA?= <101104760+ZhangSetSail@users.noreply.github.com> Date: Thu, 27 Jun 2024 15:33:56 +0800 Subject: [PATCH 24/38] perf: delete es log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 张启航 <101104760+ZhangSetSail@users.noreply.github.com> --- pkg/component/es/esComponent.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index 4e3deb1716..ea4eb40358 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -95,7 +95,5 @@ func (c *Component) request(url, method, body string) (string, error) { logrus.Errorf("Error reading response body: %s", err.Error()) return "", err } - logrus.Infof("url: %s", url) - logrus.Infof("body: %s", body) return string(data), nil } From 12a10f55a0c4abfae7df78099ae95003a26d992d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Sun, 21 Jul 2024 11:09:06 +0800 Subject: [PATCH 25/38] =?UTF-8?q?fix:=20show=20build=20use=20time=20Signed?= =?UTF-8?q?-off-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@q?= =?UTF-8?q?q.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index eff3a929d5..fedb32e802 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -21,8 +21,13 @@ type EventDaoESImpl struct { func (c *EventDaoESImpl) AddModel(mo model.Interface) error { result := mo.(*model.ServiceEvent) result.CreatedAt = time.Now().Format("2006-01-02 15:04:05") - result.StartTime = time.Now().Format("2006-01-02 15:04:05") - result.EndTime = result.StartTime + + //原本都被格式化为RFC3339重新格式化为标准的时间戳 + start, _ := time.ParseInLocation(time.RFC3339, result.StartTime, time.Local) + result.StartTime = start.Format("2006-01-02 15:04:05") + + end, _ := time.ParseInLocation(time.RFC3339, result.EndTime, time.Local) + result.EndTime = end.Format("2006-01-02 15:04:05") body, _ := json.Marshal(result) _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", result.EventID), string(body)) if err != nil { @@ -33,12 +38,18 @@ func (c *EventDaoESImpl) AddModel(mo model.Interface) error { // UpdateModel UpdateModel func (c *EventDaoESImpl) UpdateModel(mo model.Interface) error { - update := mo.(*model.ServiceEvent) - update.CreatedAt = time.Now().Format("2006-01-02 15:04:05") - update.StartTime = time.Now().Format("2006-01-02 15:04:05") - update.EndTime = update.StartTime - body, _ := json.Marshal(update) - _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", update.EventID), string(body)) + up := mo.(*model.ServiceEvent) + up.CreatedAt = time.Now().Format("2006-01-02 15:04:05") + + //原本都被格式化为RFC3339重新格式化为标准的时间戳 + start, _ := time.ParseInLocation(time.RFC3339, up.StartTime, time.Local) + up.StartTime = start.Format("2006-01-02 15:04:05") + + end, _ := time.ParseInLocation(time.RFC3339, up.EndTime, time.Local) + up.EndTime = end.Format("2006-01-02 15:04:05") + + body, _ := json.Marshal(up) + _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", up.EventID), string(body)) return err } From 22e3eb585be762ff115969745d1a37a55c4efad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Sun, 21 Jul 2024 11:59:11 +0800 Subject: [PATCH 26/38] =?UTF-8?q?fix:=20show=20build=20use=20time=20Signed?= =?UTF-8?q?-off-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@q?= =?UTF-8?q?q.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index fedb32e802..7ff736d84a 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -23,11 +23,16 @@ func (c *EventDaoESImpl) AddModel(mo model.Interface) error { result.CreatedAt = time.Now().Format("2006-01-02 15:04:05") //原本都被格式化为RFC3339重新格式化为标准的时间戳 - start, _ := time.ParseInLocation(time.RFC3339, result.StartTime, time.Local) - result.StartTime = start.Format("2006-01-02 15:04:05") + start, serr := time.ParseInLocation(time.RFC3339, result.StartTime, time.Local) + if serr == nil { + result.StartTime = start.Format("2006-01-02 15:04:05") + } + + end, eerr := time.ParseInLocation(time.RFC3339, result.EndTime, time.Local) + if eerr == nil { + result.EndTime = end.Format("2006-01-02 15:04:05") + } - end, _ := time.ParseInLocation(time.RFC3339, result.EndTime, time.Local) - result.EndTime = end.Format("2006-01-02 15:04:05") body, _ := json.Marshal(result) _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", result.EventID), string(body)) if err != nil { @@ -42,11 +47,15 @@ func (c *EventDaoESImpl) UpdateModel(mo model.Interface) error { up.CreatedAt = time.Now().Format("2006-01-02 15:04:05") //原本都被格式化为RFC3339重新格式化为标准的时间戳 - start, _ := time.ParseInLocation(time.RFC3339, up.StartTime, time.Local) - up.StartTime = start.Format("2006-01-02 15:04:05") + start, serr := time.ParseInLocation(time.RFC3339, up.StartTime, time.Local) + if serr == nil { + up.StartTime = start.Format("2006-01-02 15:04:05") + } - end, _ := time.ParseInLocation(time.RFC3339, up.EndTime, time.Local) - up.EndTime = end.Format("2006-01-02 15:04:05") + end, eerr := time.ParseInLocation(time.RFC3339, up.EndTime, time.Local) + if eerr == nil { + up.EndTime = end.Format("2006-01-02 15:04:05") + } body, _ := json.Marshal(up) _, err := es.Default().POST(fmt.Sprintf("/appstore_tenant_services_event/_doc/%s", up.EventID), string(body)) From 4f2b43a77762df9be3ee9f64172b8d4dae7ad46c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Sun, 21 Jul 2024 13:43:27 +0800 Subject: [PATCH 27/38] =?UTF-8?q?fix:=20show=20build=20use=20time=20Signed?= =?UTF-8?q?-off-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@q?= =?UTF-8?q?q.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index 7ff736d84a..b3eefa72ff 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -44,7 +44,7 @@ func (c *EventDaoESImpl) AddModel(mo model.Interface) error { // UpdateModel UpdateModel func (c *EventDaoESImpl) UpdateModel(mo model.Interface) error { up := mo.(*model.ServiceEvent) - up.CreatedAt = time.Now().Format("2006-01-02 15:04:05") + //up.CreatedAt = time.Now().Format("2006-01-02 15:04:05") //原本都被格式化为RFC3339重新格式化为标准的时间戳 start, serr := time.ParseInLocation(time.RFC3339, up.StartTime, time.Local) From eeceacddbab70d419309100c7b6b86901e2e83b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 22 Jul 2024 18:15:06 +0800 Subject: [PATCH 28/38] =?UTF-8?q?feat:=20es=20logrus=20Signed-off-by:=20?= =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/component/es/esComponent.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index ea4eb40358..3879ed7f22 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -88,6 +88,9 @@ func (c *Component) request(url, method, body string) (string, error) { if resp.StatusCode < http.StatusOK || resp.StatusCode > 300 { logrus.Error(url, body) logrus.Errorf("Error response from server: %d %s\n", resp.StatusCode, resp.Status) + data, _ := io.ReadAll(resp.Body) + logrus.Errorf("Error request body: %s\n", body) + logrus.Errorf("Error response body: %s\n", string(data)) return "", err } data, err := io.ReadAll(resp.Body) From 7a30f8f99a1f8785770ddeab16e9913d1fb03b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Mon, 22 Jul 2024 18:24:39 +0800 Subject: [PATCH 29/38] =?UTF-8?q?feat:=20es=20logrus=20Signed-off-by:=20?= =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20<1666888816@qq.com>?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mysql/dao/event_es.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index b3eefa72ff..ea5a2c0491 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -26,11 +26,15 @@ func (c *EventDaoESImpl) AddModel(mo model.Interface) error { start, serr := time.ParseInLocation(time.RFC3339, result.StartTime, time.Local) if serr == nil { result.StartTime = start.Format("2006-01-02 15:04:05") + } else { + result.StartTime = time.Now().Format("2006-01-02 15:04:05") } end, eerr := time.ParseInLocation(time.RFC3339, result.EndTime, time.Local) if eerr == nil { result.EndTime = end.Format("2006-01-02 15:04:05") + } else { + result.EndTime = time.Now().Format("2006-01-02 15:04:05") } body, _ := json.Marshal(result) @@ -50,11 +54,15 @@ func (c *EventDaoESImpl) UpdateModel(mo model.Interface) error { start, serr := time.ParseInLocation(time.RFC3339, up.StartTime, time.Local) if serr == nil { up.StartTime = start.Format("2006-01-02 15:04:05") + } else { + up.StartTime = time.Now().Format("2006-01-02 15:04:05") } end, eerr := time.ParseInLocation(time.RFC3339, up.EndTime, time.Local) if eerr == nil { up.EndTime = end.Format("2006-01-02 15:04:05") + } else { + up.EndTime = time.Now().Format("2006-01-02 15:04:05") } body, _ := json.Marshal(up) From 0ac8cf59d9cbf494dfb7bff3cceeb1df20c57282 Mon Sep 17 00:00:00 2001 From: yangk Date: Wed, 24 Jul 2024 08:06:30 +0800 Subject: [PATCH 30/38] perf: support es connect reuse Signed-off-by: yangk --- pkg/component/es/esComponent.go | 13 ++++++++----- util/k8s/k8s.go | 6 +++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index 3879ed7f22..5bbd4d0e14 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -17,6 +17,7 @@ type Component struct { url string username string password string + client *http.Client } func (c *Component) Start(ctx context.Context, cfg *configs.Config) error { @@ -30,6 +31,11 @@ func (c *Component) SingleStart(url, username, password string) { c.url = url c.username = username c.password = password + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + c.client = client } func (c *Component) CloseHandle() { @@ -70,11 +76,8 @@ func (c *Component) request(url, method, body string) (string, error) { } req.SetBasicAuth(c.username, c.password) req.Header.Set("Content-Type", "application/json") - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} - resp, err := client.Do(req) + + resp, err := c.client.Do(req) if err != nil { logrus.Errorf("Error making request: %s", err.Error()) return "", err diff --git a/util/k8s/k8s.go b/util/k8s/k8s.go index 9196d5a5fa..dec62f2020 100644 --- a/util/k8s/k8s.go +++ b/util/k8s/k8s.go @@ -56,7 +56,7 @@ func NewRestConfig(kubecfg string) (restConfig *rest.Config, err error) { return clientcmd.BuildConfigFromFlags("", kubecfg) } -//NewRestClient new rest client +// NewRestClient new rest client func NewRestClient(restConfig *rest.Config) (*rest.RESTClient, error) { return rest.RESTClientFor(restConfig) } @@ -146,7 +146,7 @@ func GetKubeVersion() *utilversion.Version { return utilversion.MustParseSemantic(serverVersion.GitVersion) } -//GetClientSet - +// GetClientSet - func GetClientSet() kubernetes.Interface { if clientset == nil { once.Do(func() { @@ -157,7 +157,7 @@ func GetClientSet() kubernetes.Interface { return clientset } -//MustNewKubeConfig - +// MustNewKubeConfig - func MustNewKubeConfig(kubeconfigPath string) *rest.Config { if kubeconfigPath != "" { cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) From 83196338c1cba93e5c218d957803beafd2078a7a Mon Sep 17 00:00:00 2001 From: yangk Date: Wed, 24 Jul 2024 08:24:39 +0800 Subject: [PATCH 31/38] fix: es connect reuse error Signed-off-by: yangk --- pkg/component/es/esComponent.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index 5bbd4d0e14..2092752dd6 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -24,6 +24,11 @@ func (c *Component) Start(ctx context.Context, cfg *configs.Config) error { c.url = cfg.APIConfig.ElasticSearchURL c.username = cfg.APIConfig.ElasticSearchUsername c.password = cfg.APIConfig.ElasticSearchPassword + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + c.client = client return nil } From 47f8782c6c072e1238e2d1d308ffb7e31d8f0f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Wed, 24 Jul 2024 12:02:34 +0800 Subject: [PATCH 32/38] =?UTF-8?q?feat:=20download=20chart=20defer=20wg.Don?= =?UTF-8?q?e()=20Signed-off-by:=20=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/helm/update.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/helm/update.go b/pkg/helm/update.go index 37948135d3..c84b1f3349 100644 --- a/pkg/helm/update.go +++ b/pkg/helm/update.go @@ -20,6 +20,7 @@ func updateCharts(repos []*repo.ChartRepository, out io.Writer, failOnRepoUpdate for _, re := range repos { wg.Add(1) _ = gogo.Go(func(ctx context.Context) error { + defer wg.Done() if _, err := re.DownloadIndexFile(); err != nil { repoFailList = append(repoFailList, re.Config.URL) } From 4bf94da2d86ad2e269eafe5256317ed053403abe Mon Sep 17 00:00:00 2001 From: yangk Date: Thu, 25 Jul 2024 09:14:53 +0800 Subject: [PATCH 33/38] perf: support configrue es and http connections Signed-off-by: yangk --- db/mysql/mysql.go | 34 +++++++++++++++++++++++++- pkg/component/es/esComponent.go | 43 +++++++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/db/mysql/mysql.go b/db/mysql/mysql.go index 03a7475a5e..c6fa278a0f 100644 --- a/db/mysql/mysql.go +++ b/db/mysql/mysql.go @@ -20,7 +20,9 @@ package mysql import ( "os" + "strconv" "sync" + "time" "github.com/goodrain/rainbond/db/config" "github.com/goodrain/rainbond/db/model" @@ -52,7 +54,37 @@ func CreateManager(config config.Config) (*Manager, error) { if err != nil { return nil, err } - + // 获取底层的 sql.DB 对象 + sqlDB := db.DB() + if err != nil { + logrus.Errorf("failed to get sql.DB from gorm.DB: %v", err) + return nil, err + } + maxOpenConns := 2500 + maxIdleConns := 500 + maxLifeTime := 5 + if os.Getenv("DB_MAX_OPEN_CONNS") != "" { + openCon, err := strconv.Atoi(os.Getenv("DB_MAX_OPEN_CONNS")) + if err == nil { + maxOpenConns = openCon + } + } + if os.Getenv("DB_MAX_IDLE_CONNS") != "" { + idleCon, err := strconv.Atoi(os.Getenv("DB_MAX_IDLE_CONNS")) + if err == nil { + maxIdleConns = idleCon + } + } + if os.Getenv("DB_CONN_MAX_LIFE_TIME") != "" { + lifeTime, err := strconv.Atoi(os.Getenv("DB_CONN_MAX_LIFE_TIME")) + if err == nil { + maxLifeTime = lifeTime + } + } + // 配置连接池参数 + sqlDB.SetMaxOpenConns(maxOpenConns) // 设置最大打开连接数 + sqlDB.SetMaxIdleConns(maxIdleConns) // 设置最大空闲连接数 + sqlDB.SetConnMaxLifetime(time.Duration(maxLifeTime) * time.Minute) // } if config.DBType == "cockroachdb" { var err error diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index 2092752dd6..a3829ab411 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -8,6 +8,9 @@ import ( "github.com/sirupsen/logrus" "io" "net/http" + "os" + "strconv" + "time" ) var defaultEsComponent *Component @@ -20,15 +23,39 @@ type Component struct { client *http.Client } +// createHTTPClient - 创建带有连接池配置的 HTTP 客户端 +func createHTTPClient() *http.Client { + maxIdleConns := 100 + if os.Getenv("HTTP_MAX_IDLE_CONNS") != "" { + idleCon, err := strconv.Atoi(os.Getenv("HTTP_MAX_IDLE_CONNS")) + if err == nil { + maxIdleConns = idleCon + } + } + maxIdleConnsPerHost := 100 + if os.Getenv("HTTP_MAX_IDLE_CONNS_PER_HOST") != "" { + idleCon, err := strconv.Atoi(os.Getenv("HTTP_MAX_IDLE_CONNS_PER_HOST")) + if err == nil { + maxIdleConnsPerHost = idleCon + } + } + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + MaxIdleConns: maxIdleConns, // 设置最大空闲连接数 + MaxIdleConnsPerHost: maxIdleConnsPerHost, // 设置每个主机的最大空闲连接数 + IdleConnTimeout: 90 * time.Second, // 设置空闲连接的超时时间 + TLSHandshakeTimeout: 10 * time.Second, // 设置 TLS 握手超时时间 + ExpectContinueTimeout: 1 * time.Second, // 设置 100-continue 状态码超时时间 + } + client := &http.Client{Transport: tr} + return client +} + func (c *Component) Start(ctx context.Context, cfg *configs.Config) error { c.url = cfg.APIConfig.ElasticSearchURL c.username = cfg.APIConfig.ElasticSearchUsername c.password = cfg.APIConfig.ElasticSearchPassword - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} - c.client = client + c.client = createHTTPClient() return nil } @@ -36,11 +63,7 @@ func (c *Component) SingleStart(url, username, password string) { c.url = url c.username = username c.password = password - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} - c.client = client + c.client = createHTTPClient() } func (c *Component) CloseHandle() { From 6de0699175d35eccc46677947d8bc24f57e20018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Fri, 26 Jul 2024 15:26:52 +0800 Subject: [PATCH 34/38] feat: add es --- db/mysql/dao_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/mysql/dao_impl.go b/db/mysql/dao_impl.go index 4d16f28e3c..357c9ca3ee 100644 --- a/db/mysql/dao_impl.go +++ b/db/mysql/dao_impl.go @@ -327,7 +327,7 @@ func (m *Manager) CodeCheckResultDaoTransactions(db *gorm.DB) dao.CodeCheckResul // ServiceEventDao TenantServicePluginRelationDao func (m *Manager) ServiceEventDao() dao.EventDao { - return &mysqldao.EventDaoESImpl{ + return &mysqldao.EventDaoImpl{ DB: m.db, } } From b09a72c6fe3318dcf21dd9a0615e0aea6a8d0c2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Fri, 26 Jul 2024 15:29:51 +0800 Subject: [PATCH 35/38] feat: add es --- db/mysql/dao/event_es.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/db/mysql/dao/event_es.go b/db/mysql/dao/event_es.go index ea5a2c0491..f2a61e96ef 100644 --- a/db/mysql/dao/event_es.go +++ b/db/mysql/dao/event_es.go @@ -13,6 +13,7 @@ import ( "time" ) +// EventDaoESImpl - type EventDaoESImpl struct { DB *gorm.DB } @@ -515,7 +516,7 @@ func (c *EventDaoESImpl) DelAbnormalEvent(serviceID, Opt string) error { // DelAllAbnormalEvent delete all Abnormal event in components when stop. func (c *EventDaoESImpl) DelAllAbnormalEvent(serviceID string, Opts []string) error { - optsJson, _ := json.Marshal(Opts) + optsJSON, _ := json.Marshal(Opts) body := fmt.Sprintf(`{ "query": { "bool": { @@ -527,7 +528,7 @@ func (c *EventDaoESImpl) DelAllAbnormalEvent(serviceID string, Opts []string) er ] } } -}`, model.TargetTypePod, serviceID, string(optsJson), model.EventStatusFailure.String()) +}`, model.TargetTypePod, serviceID, string(optsJSON), model.EventStatusFailure.String()) _, err := es.Default().POST("/appstore_tenant_services_event/_delete_by_query", body) if err != nil { return err @@ -548,7 +549,7 @@ func (c *EventDaoESImpl) SetEventStatus(ctx context.Context, status model.EventS // GetExceptionEventsByTime - func (c *EventDaoESImpl) GetExceptionEventsByTime(eventTypes []string, createTime time.Time) ([]*model.ServiceEvent, error) { - eventTypesJson, _ := json.Marshal(eventTypes) + eventTypesJSON, _ := json.Marshal(eventTypes) body := fmt.Sprintf(`{ "query": { "bool": { @@ -558,7 +559,7 @@ func (c *EventDaoESImpl) GetExceptionEventsByTime(eventTypes []string, createTim ] } } -}`, string(eventTypesJson), createTime.Format("2006-01-02 15:04:05")) +}`, string(eventTypesJSON), createTime.Format("2006-01-02 15:04:05")) return c.array(body) } From da4b425ccdd00236e3611ce1623e1a6612a9330f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Fri, 26 Jul 2024 15:31:58 +0800 Subject: [PATCH 36/38] feat: add es --- db/mysql/dao_impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/mysql/dao_impl.go b/db/mysql/dao_impl.go index 357c9ca3ee..99dca2bcea 100644 --- a/db/mysql/dao_impl.go +++ b/db/mysql/dao_impl.go @@ -402,7 +402,7 @@ func (m *Manager) AppDao() dao.AppDao { } } -// AppDao app export and import info +// KeyValueDao app export and import info func (m *Manager) KeyValueDao() dao.KeyValueDao { return &mysqldao.KeyValueImpl{ DB: m.db, From 23ac7ac83d51e8e596107dd26a9cbea29f4afc2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Fri, 26 Jul 2024 15:33:42 +0800 Subject: [PATCH 37/38] feat: add es --- pkg/component/es/esComponent.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/component/es/esComponent.go b/pkg/component/es/esComponent.go index a3829ab411..aa2a31ee3a 100644 --- a/pkg/component/es/esComponent.go +++ b/pkg/component/es/esComponent.go @@ -51,6 +51,7 @@ func createHTTPClient() *http.Client { return client } +// Start - func (c *Component) Start(ctx context.Context, cfg *configs.Config) error { c.url = cfg.APIConfig.ElasticSearchURL c.username = cfg.APIConfig.ElasticSearchUsername @@ -59,6 +60,7 @@ func (c *Component) Start(ctx context.Context, cfg *configs.Config) error { return nil } +// SingleStart - func (c *Component) SingleStart(url, username, password string) { c.url = url c.username = username @@ -66,6 +68,7 @@ func (c *Component) SingleStart(url, username, password string) { c.client = createHTTPClient() } +// CloseHandle - func (c *Component) CloseHandle() { } @@ -80,18 +83,22 @@ func Default() *Component { return defaultEsComponent } +// GET - func (c *Component) GET(url string) (string, error) { return c.request(url, "GET", "") } +// POST - func (c *Component) POST(url, body string) (string, error) { return c.request(url, "POST", body) } +// PUT - func (c *Component) PUT(url, body string) (string, error) { return c.request(url, "PUT", body) } +// DELETE - func (c *Component) DELETE(url string) (string, error) { return c.request(url, "DELETE", "") } From 30fbdb4f5216e3a5cc34d4b3aafdf407e9398823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=80=86=E6=B5=81=E8=80=8C=E4=B8=8A?= <1666888816@qq.com> Date: Fri, 26 Jul 2024 15:53:21 +0800 Subject: [PATCH 38/38] feat: args es-enable --- cmd/api/main.go | 7 ++++++- cmd/api/option/option.go | 2 ++ cmd/builder/option/option.go | 2 ++ cmd/builder/server/server.go | 4 +++- cmd/eventlog/server/server.go | 5 ++++- cmd/worker/option/option.go | 2 ++ cmd/worker/server/server.go | 4 +++- db/mysql/dao_impl.go | 6 ++++++ eventlog/conf/conf.go | 1 + 9 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 9ef693ccb1..a1c77b300a 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -23,6 +23,7 @@ import ( "context" "github.com/goodrain/rainbond/config/configs" "github.com/goodrain/rainbond/pkg/component" + "github.com/goodrain/rainbond/pkg/component/es" "github.com/goodrain/rainbond/pkg/rainbond" "github.com/sirupsen/logrus" "os" @@ -46,9 +47,13 @@ func main() { AppName: "rbd-api", APIConfig: s.Config, }) + + if s.Config.ElasticEnable { + es.New().SingleStart(s.Config.ElasticSearchURL, s.Config.ElasticSearchUsername, s.Config.ElasticSearchPassword) + } + // 启动 rbd-api err := rainbond.New(context.Background(), configs.Default()). - Registry(component.ES()). Registry(component.Database()). Registry(component.Grpc()). Registry(component.Event()). diff --git a/cmd/api/option/option.go b/cmd/api/option/option.go index 9b2c1795d4..c818f9c69e 100644 --- a/cmd/api/option/option.go +++ b/cmd/api/option/option.go @@ -71,6 +71,7 @@ type Config struct { ElasticSearchURL string ElasticSearchUsername string ElasticSearchPassword string + ElasticEnable bool } // APIServer apiserver server @@ -130,6 +131,7 @@ func (a *APIServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&a.ElasticSearchURL, "es-url", "http://47.92.106.114:9200", "es url") fs.StringVar(&a.ElasticSearchUsername, "es-username", "", "es username") fs.StringVar(&a.ElasticSearchPassword, "es-password", "", "es pwd") + fs.BoolVar(&a.ElasticEnable, "es-enable", false, "enable es") } // SetLog 设置log diff --git a/cmd/builder/option/option.go b/cmd/builder/option/option.go index d16b262e58..95b0898e88 100644 --- a/cmd/builder/option/option.go +++ b/cmd/builder/option/option.go @@ -63,6 +63,7 @@ type Config struct { ElasticSearchURL string ElasticSearchUsername string ElasticSearchPassword string + ElasticEnable bool } // Builder builder server @@ -113,6 +114,7 @@ func (a *Builder) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&a.ElasticSearchURL, "es-url", "http://47.92.106.114:9200", "es url") fs.StringVar(&a.ElasticSearchUsername, "es-username", "", "es username") fs.StringVar(&a.ElasticSearchPassword, "es-password", "", "es pwd") + fs.BoolVar(&a.ElasticEnable, "es-enable", false, "enable es") } diff --git a/cmd/builder/server/server.go b/cmd/builder/server/server.go index 8425982db5..95e0cce1da 100644 --- a/cmd/builder/server/server.go +++ b/cmd/builder/server/server.go @@ -52,7 +52,9 @@ func Run(s *option.Builder) error { MysqlConnectionInfo: s.Config.MysqlConnectionInfo, } - es.New().SingleStart(s.Config.ElasticSearchURL, s.Config.ElasticSearchUsername, s.Config.ElasticSearchPassword) + if s.ElasticEnable { + es.New().SingleStart(s.Config.ElasticSearchURL, s.Config.ElasticSearchUsername, s.Config.ElasticSearchPassword) + } if err := db.CreateManager(dbconfig); err != nil { return err diff --git a/cmd/eventlog/server/server.go b/cmd/eventlog/server/server.go index 1cfc2f656b..4f56fb08e9 100644 --- a/cmd/eventlog/server/server.go +++ b/cmd/eventlog/server/server.go @@ -106,6 +106,7 @@ func (s *LogServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.Conf.ElasticSearchURL, "es-url", "http://47.92.106.114:9200", "es url") fs.StringVar(&s.Conf.ElasticSearchUsername, "es-username", "", "es username") fs.StringVar(&s.Conf.ElasticSearchPassword, "es-password", "", "es pwd") + fs.BoolVar(&s.Conf.ElasticEnable, "es-enable", false, "enable es") } // InitLog 初始化log @@ -166,7 +167,9 @@ func (s *LogServer) Run() error { s.Logger.Debug("Start run server.") log := s.Logger - es.New().SingleStart(s.Conf.ElasticSearchURL, s.Conf.ElasticSearchUsername, s.Conf.ElasticSearchPassword) + if s.Conf.ElasticEnable { + es.New().SingleStart(s.Conf.ElasticSearchURL, s.Conf.ElasticSearchUsername, s.Conf.ElasticSearchPassword) + } //init new db if err := db.CreateDBManager(s.Conf.EventStore.DB); err != nil { diff --git a/cmd/worker/option/option.go b/cmd/worker/option/option.go index 763041178f..03476ccdaf 100644 --- a/cmd/worker/option/option.go +++ b/cmd/worker/option/option.go @@ -57,6 +57,7 @@ type Config struct { ElasticSearchURL string ElasticSearchUsername string ElasticSearchPassword string + ElasticEnable bool } // Helm helm configuration. @@ -112,6 +113,7 @@ func (a *Worker) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&a.ElasticSearchURL, "es-url", "http://47.92.106.114:9200", "es url") fs.StringVar(&a.ElasticSearchUsername, "es-username", "", "es username") fs.StringVar(&a.ElasticSearchPassword, "es-password", "", "es pwd") + fs.BoolVar(&a.ElasticEnable, "es-enable", false, "enable es") } // SetLog 设置log diff --git a/cmd/worker/server/server.go b/cmd/worker/server/server.go index c32836eb09..e29cf88951 100644 --- a/cmd/worker/server/server.go +++ b/cmd/worker/server/server.go @@ -57,8 +57,10 @@ func Run(s *option.Worker) error { DBType: s.Config.DBType, MysqlConnectionInfo: s.Config.MysqlConnectionInfo, } + if s.Config.ElasticEnable { + es.New().SingleStart(s.Config.ElasticSearchURL, s.Config.ElasticSearchUsername, s.Config.ElasticSearchPassword) - es.New().SingleStart(s.Config.ElasticSearchURL, s.Config.ElasticSearchUsername, s.Config.ElasticSearchPassword) + } //step 1:db manager init ,event log client init if err := db.CreateManager(dbconfig); err != nil { diff --git a/db/mysql/dao_impl.go b/db/mysql/dao_impl.go index 99dca2bcea..a9e946195d 100644 --- a/db/mysql/dao_impl.go +++ b/db/mysql/dao_impl.go @@ -21,6 +21,7 @@ package mysql import ( "github.com/goodrain/rainbond/db/dao" mysqldao "github.com/goodrain/rainbond/db/mysql/dao" + "github.com/goodrain/rainbond/pkg/component/es" "github.com/jinzhu/gorm" ) @@ -327,6 +328,11 @@ func (m *Manager) CodeCheckResultDaoTransactions(db *gorm.DB) dao.CodeCheckResul // ServiceEventDao TenantServicePluginRelationDao func (m *Manager) ServiceEventDao() dao.EventDao { + if es.Default() != nil { + return &mysqldao.EventDaoESImpl{ + DB: m.db, + } + } return &mysqldao.EventDaoImpl{ DB: m.db, } diff --git a/eventlog/conf/conf.go b/eventlog/conf/conf.go index 02511e2e56..4607b9013e 100644 --- a/eventlog/conf/conf.go +++ b/eventlog/conf/conf.go @@ -33,6 +33,7 @@ type Conf struct { ElasticSearchURL string ElasticSearchUsername string ElasticSearchPassword string + ElasticEnable bool } // WebHookConf webhook conf