diff --git a/protocol/app/app.go b/protocol/app/app.go index 990235763b..527db9b43f 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -1493,6 +1493,10 @@ func (app *App) BeginBlocker(ctx sdk.Context) (sdk.BeginBlock, error) { // EndBlocker application updates every end block func (app *App) EndBlocker(ctx sdk.Context) (sdk.EndBlock, error) { + // Measure the lag between current timestamp and the end blocker time stamp + // as an indicator of whether the node is lagging behind. + metrics.ModuleMeasureSince(metrics.EndBlocker, metrics.EndBlockerLag, ctx.BlockTime()) + ctx = ctx.WithExecMode(lib.ExecModeEndBlock) // Reset the logger for middleware. diff --git a/protocol/lib/metrics/metric_keys.go b/protocol/lib/metrics/metric_keys.go index 023179c826..46d8d3df9c 100644 --- a/protocol/lib/metrics/metric_keys.go +++ b/protocol/lib/metrics/metric_keys.go @@ -55,4 +55,10 @@ const ( DaemonGetPerpetualsPaginatedLatency = "daemon_get_perpetuals_paginated_latency" MevLatency = "mev_latency" GateWithdrawalsIfNegativeTncSubaccountSeenLatency = "gate_withdrawals_if_negative_tnc_subaccount_seen_latency" + + // Full node grpc + FullNodeGrpc = "full_node_grpc" + GrpcSendOrderbookUpdatesLatency = "grpc_send_orderbook_updates_latency" + EndBlocker = "end_blocker" + EndBlockerLag = "end_blocker_lag" ) diff --git a/protocol/streaming/grpc/grpc_streaming_manager.go b/protocol/streaming/grpc/grpc_streaming_manager.go index 7cc9beea23..60a00e7bdc 100644 --- a/protocol/streaming/grpc/grpc_streaming_manager.go +++ b/protocol/streaming/grpc/grpc_streaming_manager.go @@ -2,11 +2,13 @@ package grpc import ( "sync" + "time" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/gogoproto/proto" ocutypes "github.com/dydxprotocol/v4-chain/protocol/indexer/off_chain_updates/types" "github.com/dydxprotocol/v4-chain/protocol/lib" + "github.com/dydxprotocol/v4-chain/protocol/lib/metrics" "github.com/dydxprotocol/v4-chain/protocol/streaming/grpc/types" clobtypes "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" ) @@ -80,6 +82,12 @@ func (sm *GrpcStreamingManagerImpl) SendOrderbookUpdates( blockHeight uint32, execMode sdk.ExecMode, ) { + defer metrics.ModuleMeasureSince( + metrics.FullNodeGrpc, + metrics.GrpcSendOrderbookUpdatesLatency, + time.Now(), + ) + // Group updates by clob pair id. updates := make(map[uint32]*clobtypes.OffchainUpdates) for _, message := range offchainUpdates.Messages {