diff --git a/protocol/app/app.go b/protocol/app/app.go index 20fd36d602..679f4e3e73 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -824,7 +824,8 @@ func New( appFlags, logger, ) - app.RegisterDaemonWithHealthMonitor(app.SlinkyClient, maxDaemonUnhealthyDuration) + app.RegisterDaemonWithHealthMonitor(app.SlinkyClient.GetMarketPairHC(), maxDaemonUnhealthyDuration) + app.RegisterDaemonWithHealthMonitor(app.SlinkyClient.GetPriceHC(), maxDaemonUnhealthyDuration) } } diff --git a/protocol/daemons/pricefeed/client/constants/logger.go b/protocol/daemons/pricefeed/client/constants/logger.go index c5fc4a1352..b417adb534 100644 --- a/protocol/daemons/pricefeed/client/constants/logger.go +++ b/protocol/daemons/pricefeed/client/constants/logger.go @@ -9,6 +9,7 @@ const ( ErrorLogKey = "error" ExchangeIdLogKey = "exchangeId" MarketIdLogKey = "marketId" + MarketIdsLogKey = "marketIds" PriceLogKey = "Price" ReasonLogKey = "reason" diff --git a/protocol/daemons/slinky/client/client.go b/protocol/daemons/slinky/client/client.go index eca04745c1..0f37a1a676 100644 --- a/protocol/daemons/slinky/client/client.go +++ b/protocol/daemons/slinky/client/client.go @@ -2,10 +2,11 @@ package client import ( "context" - "cosmossdk.io/errors" "sync" "time" + "cosmossdk.io/errors" + "cosmossdk.io/log" oracleclient "github.com/skip-mev/slinky/service/clients/oracle" @@ -17,16 +18,14 @@ import ( libtime "github.com/dydxprotocol/v4-chain/protocol/lib/time" ) -// Ensure Client is HealthCheckable -var _ daemontypes.HealthCheckable = (*Client)(nil) - // Client is the daemon implementation for pulling price data from the slinky sidecar. type Client struct { - daemontypes.HealthCheckable ctx context.Context cf context.CancelFunc marketPairFetcher MarketPairFetcher + marketPairHC daemontypes.HealthCheckable priceFetcher PriceFetcher + priceHC daemontypes.HealthCheckable wg sync.WaitGroup logger log.Logger } @@ -34,8 +33,13 @@ type Client struct { func newClient(ctx context.Context, logger log.Logger) *Client { logger = logger.With(log.ModuleKey, SlinkyClientDaemonModuleName) client := &Client{ - HealthCheckable: daemontypes.NewTimeBoundedHealthCheckable( - SlinkyClientDaemonModuleName, + marketPairHC: daemontypes.NewTimeBoundedHealthCheckable( + SlinkyClientMarketPairFetcherDaemonModuleName, + &libtime.TimeProviderImpl{}, + logger, + ), + priceHC: daemontypes.NewTimeBoundedHealthCheckable( + SlinkyClientPriceFetcherDaemonModuleName, &libtime.TimeProviderImpl{}, logger, ), @@ -45,6 +49,14 @@ func newClient(ctx context.Context, logger log.Logger) *Client { return client } +func (c *Client) GetMarketPairHC() daemontypes.HealthCheckable { + return c.marketPairHC +} + +func (c *Client) GetPriceHC() daemontypes.HealthCheckable { + return c.priceHC +} + // start creates the main goroutines of the Client. func (c *Client) start( slinky oracleclient.OracleClient, @@ -90,9 +102,9 @@ func (c *Client) RunPriceFetcher(ctx context.Context) { err := c.priceFetcher.FetchPrices(ctx) if err != nil { c.logger.Error("Failed to run fetch prices for slinky daemon", "error", err) - c.ReportFailure(errors.Wrap(err, "failed to run PriceFetcher for slinky daemon")) + c.priceHC.ReportFailure(errors.Wrap(err, "failed to run PriceFetcher for slinky daemon")) } else { - c.ReportSuccess() + c.priceHC.ReportSuccess() } case <-ctx.Done(): return @@ -124,9 +136,9 @@ func (c *Client) RunMarketPairFetcher(ctx context.Context, appFlags appflags.Fla err = c.marketPairFetcher.FetchIdMappings(ctx) if err != nil { c.logger.Error("Failed to run fetch id mappings for slinky daemon", "error", err) - c.ReportFailure(errors.Wrap(err, "failed to run FetchIdMappings for slinky daemon")) + c.marketPairHC.ReportFailure(errors.Wrap(err, "failed to run FetchIdMappings for slinky daemon")) } else { - c.ReportSuccess() + c.marketPairHC.ReportSuccess() } case <-ctx.Done(): return diff --git a/protocol/daemons/slinky/client/client_test.go b/protocol/daemons/slinky/client/client_test.go index 3c52eb339e..7f39835918 100644 --- a/protocol/daemons/slinky/client/client_test.go +++ b/protocol/daemons/slinky/client/client_test.go @@ -73,7 +73,7 @@ func TestClient(t *testing.T) { ) waitTime := time.Second * 5 require.Eventually(t, func() bool { - return cli.HealthCheck() == nil + return cli.GetMarketPairHC().HealthCheck() == nil && cli.GetPriceHC().HealthCheck() == nil }, waitTime, time.Millisecond*500, "Slinky daemon failed to become healthy within %s", waitTime) cli.Stop() } diff --git a/protocol/daemons/slinky/client/constants.go b/protocol/daemons/slinky/client/constants.go index f69c43c84e..aa711e69b9 100644 --- a/protocol/daemons/slinky/client/constants.go +++ b/protocol/daemons/slinky/client/constants.go @@ -16,5 +16,7 @@ var ( const ( // SlinkyClientDaemonModuleName is the module name used in logging. - SlinkyClientDaemonModuleName = "slinky-client-daemon" + SlinkyClientDaemonModuleName = "slinky-client-daemon" + SlinkyClientPriceFetcherDaemonModuleName = "slinky-client-price-fetcher-daemon" + SlinkyClientMarketPairFetcherDaemonModuleName = "slinky-client-market-pair-fetcher-daemon" ) diff --git a/protocol/daemons/types/health_checkable.go b/protocol/daemons/types/health_checkable.go index 92c283c2c4..b1d344fb2b 100644 --- a/protocol/daemons/types/health_checkable.go +++ b/protocol/daemons/types/health_checkable.go @@ -1,11 +1,12 @@ package types import ( - "cosmossdk.io/log" "fmt" - libtime "github.com/dydxprotocol/v4-chain/protocol/lib/time" "sync" "time" + + "cosmossdk.io/log" + libtime "github.com/dydxprotocol/v4-chain/protocol/lib/time" ) const ( diff --git a/protocol/x/perpetuals/keeper/perpetual.go b/protocol/x/perpetuals/keeper/perpetual.go index 44f9b1ef95..4e8460b466 100644 --- a/protocol/x/perpetuals/keeper/perpetual.go +++ b/protocol/x/perpetuals/keeper/perpetual.go @@ -523,17 +523,14 @@ func (k Keeper) sampleAllPerpetuals(ctx sdk.Context) ( marketIdToIndexPrice := k.pricesKeeper.GetMarketIdToValidIndexPrice(ctx) + invalidPerpetualIndexPrices := []uint32{} for _, perp := range allPerpetuals { indexPrice, exists := marketIdToIndexPrice[perp.Params.MarketId] // Valid index price is missing if !exists { // Only log and increment stats if height is passed initialization period. if ctx.BlockHeight() > pricestypes.PriceDaemonInitializationBlocks { - log.ErrorLog( - ctx, - "Perpetual does not have valid index price. Skipping premium", - constants.MarketIdLogKey, perp.Params.MarketId, - ) + invalidPerpetualIndexPrices = append(invalidPerpetualIndexPrices, perp.Params.MarketId) telemetry.IncrCounterWithLabels( []string{ types.ModuleName, @@ -601,6 +598,13 @@ func (k Keeper) sampleAllPerpetuals(ctx sdk.Context) ( ), ) } + if len(invalidPerpetualIndexPrices) > 0 { + log.ErrorLog( + ctx, + "Perpetuals do not have valid index price. Skipping premium", + constants.MarketIdsLogKey, invalidPerpetualIndexPrices, + ) + } return samples, nil } diff --git a/protocol/x/prices/keeper/update_price.go b/protocol/x/prices/keeper/update_price.go index c7621e5c6b..b4851b5c43 100644 --- a/protocol/x/prices/keeper/update_price.go +++ b/protocol/x/prices/keeper/update_price.go @@ -60,6 +60,7 @@ func (k Keeper) GetValidMarketPriceUpdates( // 3. Collect all "valid" price updates. updates := make([]*types.MsgUpdateMarketPrices_MarketPrice, 0, len(allMarketParamPrices)) + nonExistentMarkets := []uint32{} for _, marketParamPrice := range allMarketParamPrices { marketId := marketParamPrice.Param.Id indexPrice, indexPriceExists := allIndexPrices[marketId] @@ -73,15 +74,18 @@ func (k Keeper) GetValidMarketPriceUpdates( // there will be a delay in populating index prices after network genesis or a network restart, or when a // market is created, it takes the daemon some time to warm up. if !k.IsRecentlyAvailable(ctx, marketId) { - log.ErrorLog( - ctx, - "Index price for market does not exist", - constants.MarketIdLogKey, - marketId, - ) + nonExistentMarkets = append(nonExistentMarkets, marketId) } continue } + if len(nonExistentMarkets) > 0 { + log.ErrorLog( + ctx, + "Index price for markets does not exist", + constants.MarketIdsLogKey, + nonExistentMarkets, + ) + } // Index prices of 0 are unexpected. In this scenario, we skip the proposal logic for the market and report an // error.