diff --git a/monitoring/asset_balances_collector.go b/monitoring/asset_balances_collector.go index 462711c6d..14a609c26 100644 --- a/monitoring/asset_balances_collector.go +++ b/monitoring/asset_balances_collector.go @@ -72,7 +72,7 @@ func (a *assetBalancesCollector) Collect(ch chan<- prometheus.Metric) { a.collectMx.Lock() defer a.collectMx.Unlock() - ctxdb, cancel := context.WithTimeout(context.Background(), dbTimeout) + ctxdb, cancel := context.WithTimeout(context.Background(), promTimeout) defer cancel() assets, err := a.cfg.AssetStore.FetchAllAssets(ctxdb, false, false, nil) diff --git a/monitoring/asset_collector.go b/monitoring/asset_collector.go index 83e608746..efd1f55ec 100644 --- a/monitoring/asset_collector.go +++ b/monitoring/asset_collector.go @@ -97,7 +97,7 @@ func (a *universeStatsCollector) Collect(ch chan<- prometheus.Metric) { a.collectMx.Lock() defer a.collectMx.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), dbTimeout) + ctx, cancel := context.WithTimeout(context.Background(), promTimeout) defer cancel() universeStats, err := a.cfg.UniverseStats.AggregateSyncStats(ctx) diff --git a/monitoring/config.go b/monitoring/config.go index 53dd37381..5b2202642 100644 --- a/monitoring/config.go +++ b/monitoring/config.go @@ -1,6 +1,8 @@ package monitoring import ( + "time" + "github.com/lightninglabs/taproot-assets/tapdb" "github.com/lightninglabs/taproot-assets/tapgarden" "github.com/lightninglabs/taproot-assets/universe" @@ -10,6 +12,8 @@ import ( // PrometheusConfig is the set of configuration data that specifies if // Prometheus metric exporting is activated, and if so the listening address of // the Prometheus server. +// +// nolint: lll type PrometheusConfig struct { // Active, if true, then Prometheus metrics will be exported. Active bool `long:"active" description:"if true prometheus metrics will be exported"` @@ -18,6 +22,11 @@ type PrometheusConfig struct { // main Prometheus server to scrape our metrics. ListenAddr string `long:"listenaddr" description:"the interface we should listen on for prometheus"` + // CollectorRPCTimeout is the context timeout to be used by the RPC + // calls performed during metrics collection. This should not be greater + // than the scrape interval of prometheus. + CollectorRPCTimeout time.Duration `long:"collector-rpc-timeout" description:"the default timeout to be used in the RPC calls performed during metric collection"` + // RPCServer is a pointer to the main RPC server. We use this to export // generic RPC metrics to monitor the health of the service. RPCServer *grpc.Server @@ -45,7 +54,8 @@ type PrometheusConfig struct { // metrics exporter. func DefaultPrometheusConfig() PrometheusConfig { return PrometheusConfig{ - ListenAddr: "127.0.0.1:8989", - Active: false, + ListenAddr: "127.0.0.1:8989", + Active: false, + CollectorRPCTimeout: defaultTimeout, } } diff --git a/monitoring/db_collector.go b/monitoring/db_collector.go index cb3826199..e7b67f71e 100644 --- a/monitoring/db_collector.go +++ b/monitoring/db_collector.go @@ -84,7 +84,7 @@ func (a *dbCollector) Collect(ch chan<- prometheus.Metric) { a.collectMx.Lock() defer a.collectMx.Unlock() - ctxdb, cancel := context.WithTimeout(context.Background(), dbTimeout) + ctxdb, cancel := context.WithTimeout(context.Background(), promTimeout) defer cancel() // Fetch the db size. diff --git a/monitoring/prometheus.go b/monitoring/prometheus.go index bbb8e579b..31868e96b 100644 --- a/monitoring/prometheus.go +++ b/monitoring/prometheus.go @@ -12,15 +12,18 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) +const ( + // defaultTimeout is the default timeout. + defaultTimeout = 25 * time.Second +) + var ( // serverMetrics is a global variable that holds the Prometheus metrics // for the gRPC server. serverMetrics *grpc_prometheus.ServerMetrics -) -const ( - // dbTimeout is the default database timeout. - dbTimeout = 20 * time.Second + // promTimeout is the timeout used by the prometheus collectors. + promTimeout time.Duration ) // PrometheusExporter is a metric exporter that uses Prometheus directly. The @@ -46,6 +49,11 @@ func (p *PrometheusExporter) Start() error { return fmt.Errorf("server metrics not set") } + promTimeout = defaultTimeout + if p.config.CollectorRPCTimeout.Seconds() != 0 { + promTimeout = p.config.CollectorRPCTimeout + } + // Create a custom Prometheus registry. p.registry = prometheus.NewRegistry() p.registry.MustRegister(collectors.NewProcessCollector( diff --git a/sample-tapd.conf b/sample-tapd.conf index 25b63ec8e..ba91f742b 100644 --- a/sample-tapd.conf +++ b/sample-tapd.conf @@ -346,6 +346,9 @@ ; The interface we should listen on for prometheus ; prometheus.listenaddr=127.0.0.1:8989 +; The default timeout used in prometheus collectors. +; prometheus.collector-rpc-timeout=25s + ; Enable additional histogram to track gRPC call processing performance ; (latency, etc) ; prometheus.perfhistograms=false