Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat (telemetry) Handling synchronization when registering prometheus metrics #113

Merged
merged 11 commits into from
Jul 16, 2024
Merged

feat (telemetry) Handling synchronization when registering prometheus metrics #113

Changes from 6 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d8d4817
Handling synchronization when registering prometheus metrics
Jul 15, 2024
3343b80
use sync.Map and lint
Jul 15, 2024
0711ac5
fix linting
Jul 15, 2024
146447a
use RWMap
Jul 15, 2024
81f5381
Using rwmap to cache metrics
Jul 15, 2024
06533b9
refactor
Jul 16, 2024
89f2006
Use lock for registration
Jul 16, 2024
cbda7a7
noop
Jul 16, 2024
18e693c
renaming
Jul 16, 2024
b5addbe
renaming
Jul 16, 2024
4b8a10a
renaming
Jul 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
224 changes: 135 additions & 89 deletions telemetry/prometheus/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
package prometheus

import (
"errors"
"fmt"
"strings"
"time"
"unicode"

"github.com/berachain/offchain-sdk/tools/rwstore"
"github.com/prometheus/client_golang/prometheus"
)

Expand All @@ -27,10 +29,10 @@ const (
type metrics struct {
cfg *Config

gaugeVecs map[string]*prometheus.GaugeVec
counterVecs map[string]*prometheus.CounterVec
histogramVecs map[string]*prometheus.HistogramVec
summaryVecs map[string]*prometheus.SummaryVec
gaugeVecs *rwstore.RWMap[string, *prometheus.GaugeVec]
counterVecs *rwstore.RWMap[string, *prometheus.CounterVec]
gordonbear marked this conversation as resolved.
Show resolved Hide resolved
histogramVecs *rwstore.RWMap[string, *prometheus.HistogramVec]
summaryVecs *rwstore.RWMap[string, *prometheus.SummaryVec]
}

// NewMetrics initializes a new instance of Prometheus metrics.
Expand All @@ -45,10 +47,10 @@ func NewMetrics(cfg *Config) (*metrics, error) { //nolint:revive // only used as
return p, nil
}

p.gaugeVecs = make(map[string]*prometheus.GaugeVec, initialVecCapacity)
p.counterVecs = make(map[string]*prometheus.CounterVec, initialVecCapacity)
p.histogramVecs = make(map[string]*prometheus.HistogramVec, initialVecCapacity)
p.summaryVecs = make(map[string]*prometheus.SummaryVec, initialVecCapacity)
p.gaugeVecs = rwstore.NewRWMap[string, *prometheus.GaugeVec]()
p.counterVecs = rwstore.NewRWMap[string, *prometheus.CounterVec]()
p.histogramVecs = rwstore.NewRWMap[string, *prometheus.HistogramVec]()
p.summaryVecs = rwstore.NewRWMap[string, *prometheus.SummaryVec]()
return p, nil
}

Expand All @@ -64,17 +66,8 @@ func (p *metrics) Gauge(name string, value float64, _ float64, tags ...string) {

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
gaugeVec, exists := p.gaugeVecs[name]
if !exists {
gaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " gauge",
}, labels)
prometheus.MustRegister(gaugeVec)
p.gaugeVecs[name] = gaugeVec
}

gaugeVec := p.mustGetOrCreateGaugeVec(name, labels)
gaugeVec.WithLabelValues(labelValues...).Set(value)
}

Expand All @@ -86,17 +79,13 @@ func (p *metrics) Incr(name string, tags ...string) {

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
gaugeVec, exists := p.gaugeVecs[name]
if !exists {
gaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " incr/decr gauge",
}, labels)
prometheus.MustRegister(gaugeVec)
p.gaugeVecs[name] = gaugeVec

if gaugeVec, exists := p.gaugeVecs.Get(name); exists {
gaugeVec.WithLabelValues(labelValues...).Inc()
return
}

gaugeVec := p.mustGetOrCreateGaugeVec(name, labels)
gaugeVec.WithLabelValues(labelValues...).Inc()
}

Expand All @@ -108,17 +97,8 @@ func (p *metrics) Decr(name string, tags ...string) {

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
gaugeVec, exists := p.gaugeVecs[name]
if !exists {
gaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " incr/decr gauge",
}, labels)
prometheus.MustRegister(gaugeVec)
p.gaugeVecs[name] = gaugeVec
}

gaugeVec := p.mustGetOrCreateGaugeVec(name, labels)
gaugeVec.WithLabelValues(labelValues...).Dec()
}

Expand All @@ -130,17 +110,8 @@ func (p *metrics) Count(name string, value int64, tags ...string) {

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
counterVec, exists := p.counterVecs[name]
if !exists {
counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " counter",
}, labels)
prometheus.MustRegister(counterVec)
p.counterVecs[name] = counterVec
}

counterVec := p.mustGetOrCreateCounterVec(name, labels)
counterVec.WithLabelValues(labelValues...).Add(float64(value))
}

Expand All @@ -152,17 +123,8 @@ func (p *metrics) IncMonotonic(name string, tags ...string) {

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
counterVec, exists := p.counterVecs[name]
if !exists {
counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " counter",
}, labels)
prometheus.MustRegister(counterVec)
p.counterVecs[name] = counterVec
}

counterVec := p.mustGetOrCreateCounterVec(name, labels)
counterVec.WithLabelValues(labelValues...).Inc()
}

Expand All @@ -181,19 +143,35 @@ func (p *metrics) Histogram(name string, value float64, rate float64, tags ...st

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
histogramVec, exists := p.histogramVecs[name]
if !exists {
histogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " histogram",
// The maximum covered stats range is rate * HistogramBucketCount
Buckets: prometheus.LinearBuckets(0, rate, p.cfg.HistogramBucketCount),
}, labels)
prometheus.MustRegister(histogramVec)
p.histogramVecs[name] = histogramVec

if histogramVec, exists := p.histogramVecs.Get(name); exists {
histogramVec.WithLabelValues(labelValues...).Observe(value)
return
}

histogramVec := prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " histogram",
// The maximum covered stats range is rate * HistogramBucketCount
Buckets: prometheus.LinearBuckets(0, rate, p.cfg.HistogramBucketCount),
}, labels)

if err := prometheus.Register(histogramVec); err != nil {
// In case of concurrent registration, get the one that has already registered
var alreadyRegisteredError prometheus.AlreadyRegisteredError
if errors.As(err, &alreadyRegisteredError) {
//nolint:errcheck // OK
histogramVec = alreadyRegisteredError.ExistingCollector.(*prometheus.HistogramVec)
} else {
// Otherwise we should panic to fail fast
panic(err)
}
} else {
p.histogramVecs.Set(name, histogramVec)
}

histogramVec.WithLabelValues(labelValues...).Observe(value)
}

Expand All @@ -206,23 +184,36 @@ func (p *metrics) Time(name string, value time.Duration, tags ...string) {

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
summaryVec, exists := p.summaryVecs[name]
if !exists {
summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " timing summary",
Objectives: map[float64]float64{
quantile50: errorMargin50,
quantile90: errorMargin90,
quantile99: errorMargin99,
},
}, labels)
prometheus.MustRegister(summaryVec)
p.summaryVecs[name] = summaryVec
summaryVec, ok := p.summaryVecs.Get(name)
if ok {
summaryVec.WithLabelValues(labels...).Observe(value.Seconds())
return
}

summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " timing summary",
Objectives: map[float64]float64{
quantile50: errorMargin50,
quantile90: errorMargin90,
quantile99: errorMargin99,
},
}, labels)
if err := prometheus.Register(summaryVec); err != nil {
// In case of concurrent registration, get the one that has already registered
var alreadyRegisteredError prometheus.AlreadyRegisteredError
if errors.As(err, &alreadyRegisteredError) {
//nolint:errcheck // OK
summaryVec = alreadyRegisteredError.ExistingCollector.(*prometheus.SummaryVec)
} else {
// Otherwise we should panic to fail fast
panic(err)
}
} else {
p.summaryVecs.Set(name, summaryVec)
}
// Convert time.Duration to seconds since Prometheus prefers base units
// see https://prometheus.io/docs/practices/naming/#base-units
summaryVec.WithLabelValues(labelValues...).Observe(value.Seconds())
Expand Down Expand Up @@ -279,3 +270,58 @@ func setDefaultCfg(cfg *Config) {
cfg.HistogramBucketCount = DefaultBucketCount
}
}

// Helper method to get or create a GaugeVec.
func (p *metrics) mustGetOrCreateGaugeVec(name string, labels []string) *prometheus.GaugeVec {
gordonbear marked this conversation as resolved.
Show resolved Hide resolved
// Attempt to read from the RWMap without locking.
if gaugeVec, exists := p.gaugeVecs.Get(name); exists {
return gaugeVec
}

// Create a new GaugeVec.
gaugeVec := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " gauge",
}, labels)

// Register the GaugeVec or get the already registered one.
if err := prometheus.Register(gaugeVec); err != nil {
var alreadyRegisteredError prometheus.AlreadyRegisteredError
if errors.As(err, &alreadyRegisteredError) {
return alreadyRegisteredError.ExistingCollector.(*prometheus.GaugeVec)
}
panic(err) // Otherwise we should panic to fail fast
}

p.gaugeVecs.Set(name, gaugeVec)
return gaugeVec
}

func (p *metrics) mustGetOrCreateCounterVec(name string, labels []string) *prometheus.CounterVec {
// Attempt to read from the RWMap without locking.
if counterVec, exists := p.counterVecs.Get(name); exists {
return counterVec
}

// Create a new CounterVec.
counterVec := prometheus.NewCounterVec(prometheus.CounterOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " counter",
}, labels)

// Register the CounterVec or get the already registered one.
if err := prometheus.Register(counterVec); err != nil {
var alreadyRegisteredError prometheus.AlreadyRegisteredError
if errors.As(err, &alreadyRegisteredError) {
return alreadyRegisteredError.ExistingCollector.(*prometheus.CounterVec)
}
panic(err) // Otherwise we should panic to fail fast
}

p.counterVecs.Set(name, counterVec)
return counterVec
}
Loading