Metrics for grpc full node streaming (#1472) (#1474)

* Metrics for grpc full node streaming * comments (cherry picked from commit 3c493f5) Co-authored-by: jayy04 <[email protected]>
dydxprotocol · May 7, 2024 · 203198f · 203198f
1 parent 20c3a3d
commit 203198f
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 0 deletions.
diff --git a/protocol/app/app.go b/protocol/app/app.go
@@ -1493,6 +1493,10 @@ func (app *App) BeginBlocker(ctx sdk.Context) (sdk.BeginBlock, error) {
 
 // EndBlocker application updates every end block
 func (app *App) EndBlocker(ctx sdk.Context) (sdk.EndBlock, error) {
+	// Measure the lag between current timestamp and the end blocker time stamp
+	// as an indicator of whether the node is lagging behind.
+	metrics.ModuleMeasureSince(metrics.EndBlocker, metrics.EndBlockerLag, ctx.BlockTime())
+
 	ctx = ctx.WithExecMode(lib.ExecModeEndBlock)
 
 	// Reset the logger for middleware.

diff --git a/protocol/lib/metrics/metric_keys.go b/protocol/lib/metrics/metric_keys.go
@@ -55,4 +55,10 @@ const (
 	DaemonGetPerpetualsPaginatedLatency               = "daemon_get_perpetuals_paginated_latency"
 	MevLatency                                        = "mev_latency"
 	GateWithdrawalsIfNegativeTncSubaccountSeenLatency = "gate_withdrawals_if_negative_tnc_subaccount_seen_latency"
+
+	// Full node grpc
+	FullNodeGrpc                    = "full_node_grpc"
+	GrpcSendOrderbookUpdatesLatency = "grpc_send_orderbook_updates_latency"
+	EndBlocker                      = "end_blocker"
+	EndBlockerLag                   = "end_blocker_lag"
 )
diff --git a/protocol/streaming/grpc/grpc_streaming_manager.go b/protocol/streaming/grpc/grpc_streaming_manager.go
@@ -2,11 +2,13 @@ package grpc
 
 import (
 	"sync"
+	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/gogoproto/proto"
 	ocutypes "github.com/dydxprotocol/v4-chain/protocol/indexer/off_chain_updates/types"
 	"github.com/dydxprotocol/v4-chain/protocol/lib"
+	"github.com/dydxprotocol/v4-chain/protocol/lib/metrics"
 	"github.com/dydxprotocol/v4-chain/protocol/streaming/grpc/types"
 	clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types"
 )
@@ -80,6 +82,12 @@ func (sm *GrpcStreamingManagerImpl) SendOrderbookUpdates(
 	blockHeight uint32,
 	execMode sdk.ExecMode,
 ) {
+	defer metrics.ModuleMeasureSince(
+		metrics.FullNodeGrpc,
+		metrics.GrpcSendOrderbookUpdatesLatency,
+		time.Now(),
+	)
+
 	// Group updates by clob pair id.
 	updates := make(map[uint32]*clobtypes.OffchainUpdates)
 	for _, message := range offchainUpdates.Messages {