Skip to content

Commit

Permalink
feat(reporter): pass scene information to reporter (#119)
Browse files Browse the repository at this point in the history
  • Loading branch information
dumbFeng authored Sep 9, 2022
1 parent 66a4f73 commit 9588228
Show file tree
Hide file tree
Showing 20 changed files with 229 additions and 73 deletions.
6 changes: 3 additions & 3 deletions doc/zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,13 +190,13 @@ h.Stop()
### Dump事件上报

您可以通过实现`Reporter` 来实现以下功能:
* 发送告警信息,当`holmes`触发`Dump`操作时。
* 发送包含现场的告警信息,当`holmes`触发`Dump`操作时。
*`Profiles`上传到其他地方,以防实例被销毁,从而导致profile丢失,或进行分析。

```go
type ReporterImpl struct{}
func (r *ReporterImpl) Report(pType string, buf []byte, reason string, eventID string) error{
// do something
func (r *ReporterImpl) Report(pType string, filename string, reason ReasonType, eventID string, sampleTime time.Time, pprofBytes []byte, scene Scene) error{
// do something
}
......
r := &ReporterImpl{} // a implement of holmes.ProfileReporter Interface.
Expand Down
2 changes: 1 addition & 1 deletion example/1gbslice/1gbslice.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (

func init() {
http.HandleFunc("/make1gb", make1gbslice)
go http.ListenAndServe(":10003", nil)
go http.ListenAndServe(":10003", nil) //nolint:errcheck
}

func main() {
Expand Down
2 changes: 1 addition & 1 deletion example/alloc/alloc.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (

func init() {
http.HandleFunc("/alloc", alloc)
go http.ListenAndServe(":10003", nil)
go http.ListenAndServe(":10003", nil) //nolint:errcheck
}

func main() {
Expand Down
2 changes: 1 addition & 1 deletion example/channelblock/channelblock.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (

func init() {
http.HandleFunc("/chanblock", channelBlock)
go http.ListenAndServe(":10003", nil)
go http.ListenAndServe(":10003", nil) //nolint:errcheck
}

func main() {
Expand Down
2 changes: 1 addition & 1 deletion example/cpu_explode/cpu_explode.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (

func init() {
http.HandleFunc("/cpuex", cpuex)
go http.ListenAndServe(":10003", nil)
go http.ListenAndServe(":10003", nil) //nolint:errcheck
}

func main() {
Expand Down
2 changes: 1 addition & 1 deletion example/deadlock/deadlock.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func init() {
http.HandleFunc("/lockorder1", lockorder1)
http.HandleFunc("/lockorder2", lockorder2)
http.HandleFunc("/req", req)
go http.ListenAndServe(":10003", nil)
go http.ListenAndServe(":10003", nil) //nolint:errcheck
}

func main() {
Expand Down
9 changes: 6 additions & 3 deletions example/deadloop/deadloop.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (

func init() {
http.HandleFunc("/deadloop", deadloop)
go http.ListenAndServe(":10003", nil)
go http.ListenAndServe(":10003", nil) //nolint:errcheck
}

func main() {
Expand All @@ -42,8 +42,11 @@ func main() {
}

func deadloop(wr http.ResponseWriter, req *http.Request) {
for i := 0; i < 4; i++ {
for {
for {
select {
case <-req.Context().Done():
break
default:
time.Sleep(time.Millisecond)
}
}
Expand Down
19 changes: 15 additions & 4 deletions example/run_in_docker/run_in_docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func init() {
http.HandleFunc("/docker", dockermake1gb)
http.HandleFunc("/docker/cpu", cpuex)
http.HandleFunc("/docker/cpu_multi_core", cpuMulticore)
go http.ListenAndServe(":10003", nil)
go http.ListenAndServe(":10003", nil) //nolint:errcheck
}

func main() {
Expand All @@ -47,19 +47,30 @@ func main() {
}

func cpuex(wr http.ResponseWriter, req *http.Request) {
go func() {
for {
for {
select {
case <-req.Context().Done():
break
default:
time.Sleep(time.Millisecond)
}
}()
}
}

func cpuMulticore(wr http.ResponseWriter, req *http.Request) {
for i := 1; i <= 100; i++ {
go func() {
for {
select {
case <-req.Context().Done():
default:
time.Sleep(time.Millisecond)
}
}
}()
}

<-req.Context().Done()
}

func dockermake1gb(wr http.ResponseWriter, req *http.Request) {
Expand Down
2 changes: 1 addition & 1 deletion example/slowlyleak/slowlyleak.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (

func init() {
http.HandleFunc("/leak", leak)
go http.ListenAndServe(":10003", nil)
go http.ListenAndServe(":10003", nil) //nolint:errcheck
}

func main() {
Expand Down
55 changes: 41 additions & 14 deletions holmes.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,6 @@ type Holmes struct {
rptEventsCh chan rptEvent
}

type ProfileReporter interface {
Report(pType string, filename string, reason string, eventID string, sampleTime time.Time, pprofBytes []byte) error
}

// New creates a holmes dumper.
func New(opts ...Option) (*Holmes, error) {
holmes := &Holmes{
Expand Down Expand Up @@ -375,8 +371,14 @@ func (h *Holmes) goroutineProfile(gNum int, c grOptions) bool {
var buf bytes.Buffer
_ = pprof.Lookup("goroutine").WriteTo(&buf, int(h.opts.DumpProfileType)) // nolint: errcheck

scene := Scene{
typeOption: *c.typeOption,
CurVal: gNum,
Avg: h.grNumStats.avg(),
}

h.ReportProfile(type2name[goroutine], h.writeProfileDataToFile(buf, goroutine, ""),
reason, "", time.Now(), buf.Bytes())
reason, "", time.Now(), buf.Bytes(), scene)
return true
}

Expand Down Expand Up @@ -417,7 +419,13 @@ func (h *Holmes) memProfile(rss int, c typeOption) bool {
var buf bytes.Buffer
_ = pprof.Lookup("heap").WriteTo(&buf, int(h.opts.DumpProfileType)) // nolint: errcheck

h.ReportProfile(type2name[mem], h.writeProfileDataToFile(buf, mem, ""), reason, "", time.Now(), buf.Bytes())
scene := Scene{
typeOption: c,
CurVal: rss,
Avg: h.memStats.avg(),
}

h.ReportProfile(type2name[mem], h.writeProfileDataToFile(buf, mem, ""), reason, "", time.Now(), buf.Bytes(), scene)
return true
}

Expand Down Expand Up @@ -537,14 +545,20 @@ func (h *Holmes) threadProfile(curThreadNum int, c typeOption) bool {

_ = pprof.Lookup("threadcreate").WriteTo(&buf, int(h.opts.DumpProfileType)) // nolint: errcheck

scene := Scene{
typeOption: c,
CurVal: curThreadNum,
Avg: h.threadStats.avg(),
}

h.ReportProfile(type2name[thread], h.writeProfileDataToFile(buf, thread, eventID),
reason, eventID, time.Now(), buf.Bytes())
reason, eventID, time.Now(), buf.Bytes(), scene)

buf.Reset()
_ = pprof.Lookup("goroutine").WriteTo(&buf, int(h.opts.DumpProfileType)) // nolint: errcheck

h.ReportProfile(type2name[goroutine], h.writeProfileDataToFile(buf, goroutine, eventID),
reason, eventID, time.Now(), buf.Bytes())
reason, eventID, time.Now(), buf.Bytes(), scene)

return true
}
Expand Down Expand Up @@ -613,9 +627,15 @@ func (h *Holmes) cpuProfile(curCPUUsage int, c typeOption) bool {
h.Infof("[Holmes] CPU profile name : " + "::" + binFileName + " \n" + string(bfCpy))
}

scene := Scene{
typeOption: c,
CurVal: curCPUUsage,
Avg: h.cpuStats.avg(),
}

if rptOpts.active == 1 {
h.ReportProfile(type2name[cpu], binFileName,
reason, "", time.Now(), bfCpy)
reason, "", time.Now(), bfCpy, scene)
}

return true
Expand Down Expand Up @@ -731,8 +751,14 @@ func (h *Holmes) gcHeapProfile(gc int, force bool, c typeOption) bool {
var buf bytes.Buffer
_ = pprof.Lookup("heap").WriteTo(&buf, int(h.opts.DumpProfileType)) // nolint: errcheck

scene := Scene{
typeOption: c,
CurVal: gc,
Avg: h.gcHeapStats.avg(),
}

h.ReportProfile(type2name[gcHeap], h.writeProfileDataToFile(buf, gcHeap, eventID),
reason, eventID, time.Now(), buf.Bytes())
reason, eventID, time.Now(), buf.Bytes(), scene)
return true
}

Expand Down Expand Up @@ -793,9 +819,9 @@ func (h *Holmes) EnableProfileReporter() {
atomic.StoreInt32(&h.opts.rptOpts.active, 1)
}

func (h *Holmes) ReportProfile(pType string, filename string, reason string, eventID string, sampleTime time.Time, pprofBytes []byte) {
func (h *Holmes) ReportProfile(pType string, filename string, reason ReasonType, eventID string, sampleTime time.Time, pprofBytes []byte, scene Scene) {
if filename == "" {
h.Errorf("dump name is empty, type:%s, reason:%s, eventID:%s", pType, reason, eventID)
h.Errorf("dump name is empty, type:%s, reason:%s, eventID:%s", pType, reason.String(), eventID)
return
}

Expand All @@ -821,6 +847,7 @@ func (h *Holmes) ReportProfile(pType string, filename string, reason string, eve
EventID: eventID,
SampleTime: sampleTime,
PprofBytes: pprofBytes,
Scene: scene,
}

// read channel should be atomic.
Expand Down Expand Up @@ -850,9 +877,9 @@ func (h *Holmes) startReporter(ch chan rptEvent) {
}

// It's supposed to be sending judgment, isn't it?
err := opts.reporter.Report(evt.PType, evt.FileName, evt.Reason, evt.EventID, evt.SampleTime, evt.PprofBytes) // nolint: errcheck
err := opts.reporter.Report(evt.PType, evt.FileName, evt.Reason, evt.EventID, evt.SampleTime, evt.PprofBytes, evt.Scene) // nolint: errcheck
if err != nil {
h.Infof("reporter err:", err)
h.Infof("reporter err:%v", err)

}
}
Expand Down
11 changes: 6 additions & 5 deletions holmes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package holmes

import (
"log"
"os"
"runtime"
"testing"
"time"
Expand All @@ -37,15 +38,15 @@ func TestMain(m *testing.M) {
h.EnableGoroutineDump().Start()
time.Sleep(10 * time.Second)
log.Println("on running")
m.Run()
os.Exit(m.Run())
}

// -gcflags=all=-l
func TestResetCollectInterval(t *testing.T) {
before := h.collectCount
go func() {
h.Set(WithCollectInterval("2s"))
defer h.Set(WithCollectInterval("1s"))
h.Set(WithCollectInterval("2s")) //nolint:errcheck
defer h.Set(WithCollectInterval("1s")) //nolint:errcheck
time.Sleep(6 * time.Second)
// if collect interval not change, collectCount would increase 5 at least
if h.collectCount-before >= 5 {
Expand Down Expand Up @@ -74,7 +75,7 @@ func TestSetGrOpts(t *testing.T) {
}

func TestCpuCore(t *testing.T) {
h.Set(
_ = h.Set(
WithCGroup(false),
WithGoProcAsCPUCore(false),
)
Expand All @@ -96,7 +97,7 @@ func TestCpuCore(t *testing.T) {
}

// set cpu core directly
h.Set(
_ = h.Set(
WithCPUCore(cpuCore1 + 5),
)

Expand Down
10 changes: 0 additions & 10 deletions options.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,16 +67,6 @@ type options struct {
rptOpts *ReporterOptions
}

// rptEvent stands of the args of report event
type rptEvent struct {
PType string
FileName string
Reason string
EventID string
SampleTime time.Time
PprofBytes []byte
}

type ReporterOptions struct {
reporter ProfileReporter
active int32 // switch
Expand Down
4 changes: 2 additions & 2 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,12 +165,12 @@ You can use `Set` method to modify holmes' configurations when the application i
### Reporter dump event

You can use `Reporter` to implement the following features:
* Send alarm messages when holmes dump profiles.
* Send alarm messages that include the scene information when holmes dump profiles.
* Send profiles to the data center for saving or analyzing.

```go
type ReporterImpl struct{}
func (r *ReporterImpl) Report(pType string, buf []byte, reason string, eventID string) error{
func (r *ReporterImpl) Report(pType string, filename string, reason ReasonType, eventID string, sampleTime time.Time, pprofBytes []byte, scene Scene) error{
// do something
}
......
Expand Down
Loading

0 comments on commit 9588228

Please sign in to comment.