From 203198faf103287c7c4a3e6994822d84f1cb6e7c Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Tue, 7 May 2024 13:19:24 -0400
Subject: [PATCH] Metrics for grpc full node streaming (#1472) (#1474)

* Metrics for grpc full node streaming

* comments

(cherry picked from commit 3c493f5d611c10d1f9f076436aadc14bddbc5eaf)

Co-authored-by: jayy04 <103467857+jayy04@users.noreply.github.com>
---
 protocol/app/app.go                               | 4 ++++
 protocol/lib/metrics/metric_keys.go               | 6 ++++++
 protocol/streaming/grpc/grpc_streaming_manager.go | 8 ++++++++
 3 files changed, 18 insertions(+)

diff --git a/protocol/app/app.go b/protocol/app/app.go
index 990235763b..527db9b43f 100644
--- a/protocol/app/app.go
+++ b/protocol/app/app.go
@@ -1493,6 +1493,10 @@ func (app *App) BeginBlocker(ctx sdk.Context) (sdk.BeginBlock, error) {
 
 // EndBlocker application updates every end block
 func (app *App) EndBlocker(ctx sdk.Context) (sdk.EndBlock, error) {
+	// Measure the lag between current timestamp and the end blocker time stamp
+	// as an indicator of whether the node is lagging behind.
+	metrics.ModuleMeasureSince(metrics.EndBlocker, metrics.EndBlockerLag, ctx.BlockTime())
+
 	ctx = ctx.WithExecMode(lib.ExecModeEndBlock)
 
 	// Reset the logger for middleware.
diff --git a/protocol/lib/metrics/metric_keys.go b/protocol/lib/metrics/metric_keys.go
index 023179c826..46d8d3df9c 100644
--- a/protocol/lib/metrics/metric_keys.go
+++ b/protocol/lib/metrics/metric_keys.go
@@ -55,4 +55,10 @@ const (
 	DaemonGetPerpetualsPaginatedLatency               = "daemon_get_perpetuals_paginated_latency"
 	MevLatency                                        = "mev_latency"
 	GateWithdrawalsIfNegativeTncSubaccountSeenLatency = "gate_withdrawals_if_negative_tnc_subaccount_seen_latency"
+
+	// Full node grpc
+	FullNodeGrpc                    = "full_node_grpc"
+	GrpcSendOrderbookUpdatesLatency = "grpc_send_orderbook_updates_latency"
+	EndBlocker                      = "end_blocker"
+	EndBlockerLag                   = "end_blocker_lag"
 )
diff --git a/protocol/streaming/grpc/grpc_streaming_manager.go b/protocol/streaming/grpc/grpc_streaming_manager.go
index 7cc9beea23..60a00e7bdc 100644
--- a/protocol/streaming/grpc/grpc_streaming_manager.go
+++ b/protocol/streaming/grpc/grpc_streaming_manager.go
@@ -2,11 +2,13 @@ package grpc
 
 import (
 	"sync"
+	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/gogoproto/proto"
 	ocutypes "github.com/dydxprotocol/v4-chain/protocol/indexer/off_chain_updates/types"
 	"github.com/dydxprotocol/v4-chain/protocol/lib"
+	"github.com/dydxprotocol/v4-chain/protocol/lib/metrics"
 	"github.com/dydxprotocol/v4-chain/protocol/streaming/grpc/types"
 	clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types"
 )
@@ -80,6 +82,12 @@ func (sm *GrpcStreamingManagerImpl) SendOrderbookUpdates(
 	blockHeight uint32,
 	execMode sdk.ExecMode,
 ) {
+	defer metrics.ModuleMeasureSince(
+		metrics.FullNodeGrpc,
+		metrics.GrpcSendOrderbookUpdatesLatency,
+		time.Now(),
+	)
+
 	// Group updates by clob pair id.
 	updates := make(map[uint32]*clobtypes.OffchainUpdates)
 	for _, message := range offchainUpdates.Messages {