diff --git a/src/app/app.service.ts b/src/app/app.service.ts index c363dc8..79b801b 100644 --- a/src/app/app.service.ts +++ b/src/app/app.service.ts @@ -1,11 +1,12 @@ import { Inject, Injectable, LoggerService, OnModuleInit } from '@nestjs/common'; import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; -import { ConfigService } from 'common/config'; +import { ConfigService, ENV_KEYS, EnvironmentVariables } from 'common/config'; import { PrometheusService } from 'common/prometheus'; import { ConsensusProviderService } from 'common/consensus-provider'; import { ExecutionProviderService } from 'common/execution-provider'; import { APP_NAME, APP_VERSION } from './app.constants'; +import { commonPatterns, satanizer } from '@lidofinance/satanizer'; @Injectable() export class AppService implements OnModuleInit { @@ -20,14 +21,8 @@ export class AppService implements OnModuleInit { public async onModuleInit(): Promise { await this.validateNetwork(); - - const network = await this.executionProviderService.getNetworkName(); - const env = this.configService.get('NODE_ENV'); - const version = APP_VERSION; - const name = APP_NAME; - - this.prometheusService.buildInfo.labels({ env, network, name, version }).inc(); - this.logger.log('Init app', { env, network, name, version }); + await this.prometheusBuildInfoMetrics(); + this.prometheusEnvsInfoMetrics(); } /** @@ -43,4 +38,27 @@ export class AppService implements OnModuleInit { throw new Error('Chain ids do not match'); } } + + protected async prometheusBuildInfoMetrics() { + const network = await this.executionProviderService.getNetworkName(); + const env = this.configService.get('NODE_ENV'); + const version = APP_VERSION; + const name = APP_NAME; + + this.prometheusService.buildInfo.labels({ env, network, name, version }).inc(); + this.logger.log('Init app', { env, network, name, version }); + } + + protected prometheusEnvsInfoMetrics() { + const secrets = this.configService.secrets; + const mask = satanizer([...commonPatterns, ...secrets]); + + const allConfigEnvs = {}; + ENV_KEYS.forEach((key: keyof EnvironmentVariables) => { + allConfigEnvs[key] = mask(this.configService.get(key)); + }); + + this.prometheusService.envsInfo.labels(allConfigEnvs).inc(); + this.logger.log('Init app dumping envs', allConfigEnvs); + } } diff --git a/src/common/config/env.validation.ts b/src/common/config/env.validation.ts index 37ba99b..5bf2da9 100644 --- a/src/common/config/env.validation.ts +++ b/src/common/config/env.validation.ts @@ -18,7 +18,7 @@ export class EnvironmentVariables { @IsNumber() @Min(1) @Transform(toNumber({ defaultValue: 3000 })) - PORT: number; + PORT: number = 3000; @IsOptional() @IsString() @@ -49,16 +49,16 @@ export class EnvironmentVariables { @IsOptional() @IsEnum(LogLevel) @Transform(({ value }) => value || LogLevel.info) - LOG_LEVEL: LogLevel; + LOG_LEVEL: LogLevel = null; @IsOptional() @IsEnum(LogFormat) @Transform(({ value }) => value || LogFormat.json) - LOG_FORMAT: LogFormat; + LOG_FORMAT: LogFormat = null; @IsOptional() @IsString() - JOB_INTERVAL_VALIDATORS; + JOB_INTERVAL_VALIDATORS = null; @IsOptional() @IsString() @@ -71,17 +71,18 @@ export class EnvironmentVariables { @IsArray() @ArrayMinSize(1) @Transform(({ value }) => value.split(',')) - CL_API_URLS!: string[]; + CL_API_URLS: string[] = null; @IsArray() @ArrayMinSize(1) @Transform(({ value }) => value.split(',')) - EL_RPC_URLS!: string[]; + EL_RPC_URLS: string[] = null; @IsNumber() @Transform(({ value }) => Number(value)) - CHAIN_ID!: number; + CHAIN_ID: number = null; } +export const ENV_KEYS = Object.keys(new EnvironmentVariables()); export function validate(config: Record) { const validatedConfig = plainToClass(EnvironmentVariables, config); diff --git a/src/common/health/health.module.ts b/src/common/health/health.module.ts index 27c4bc2..f5a8851 100644 --- a/src/common/health/health.module.ts +++ b/src/common/health/health.module.ts @@ -1,13 +1,34 @@ -import { Module } from '@nestjs/common'; +import { Inject, LoggerService, Module, OnModuleInit } from '@nestjs/common'; import { TerminusModule } from '@nestjs/terminus'; import { HealthController } from './health.controller'; import { ExecutionProviderHealthIndicator } from './execution-provider.indicator'; import { ConsensusProviderIndicator } from './consensus-provider.indicator'; import { GenesisTimeModule } from '../genesis-time'; +import { LOGGER_PROVIDER } from '@lido-nestjs/logger'; @Module({ providers: [ExecutionProviderHealthIndicator, ConsensusProviderIndicator], controllers: [HealthController], imports: [TerminusModule, GenesisTimeModule], }) -export class HealthModule {} +export class HealthModule implements OnModuleInit { + constructor( + @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly consensusProviderIndicator: ConsensusProviderIndicator, + protected readonly executionProviderIndicator: ExecutionProviderHealthIndicator, + ) {} + + async onModuleInit() { + await this.startUpChecks(); + } + + async startUpChecks() { + try { + await this.consensusProviderIndicator.isHealthy('consensusProvider'); + await this.executionProviderIndicator.isHealthy('executionProvider'); + this.logger.log(`Start up checks passed successfully`); + } catch (e) { + this.logger.error(`Start up checks failed with error: ${e}`); + } + } +} diff --git a/src/common/prometheus/prometheus.service.ts b/src/common/prometheus/prometheus.service.ts index 304bffb..0fc7b6d 100644 --- a/src/common/prometheus/prometheus.service.ts +++ b/src/common/prometheus/prometheus.service.ts @@ -2,6 +2,7 @@ import { getOrCreateMetric } from '@willsoto/nestjs-prometheus'; import { Options, Metrics, Metric } from './interfaces'; import { METRICS_PREFIX } from './prometheus.constants'; import { RequestSourceType } from '../../http/request-time/headers/request-source-type'; +import { ENV_KEYS } from '../config'; export class PrometheusService { protected prefix = METRICS_PREFIX; @@ -25,7 +26,19 @@ export class PrometheusService { public buildInfo = this.getOrCreateMetric('Gauge', { name: 'build_info', help: 'Build information', - labelNames: ['name', 'version', 'env', 'network', 'startSlot'], + labelNames: ['name', 'version', 'env', 'network'], + }); + + public envsInfo = this.getOrCreateMetric('Gauge', { + name: METRICS_PREFIX + 'envs_info', + help: 'Environment variables information', + labelNames: ENV_KEYS, + }); + + public validatorsState = this.getOrCreateMetric('Gauge', { + name: METRICS_PREFIX + 'validators_state', + help: 'balances of Lido validators with withdrawable_epoch by frames', + labelNames: ['frame', 'balance'], }); public clApiRequestDuration = this.getOrCreateMetric('Histogram', { diff --git a/src/common/validators/strigify-frame-balances.ts b/src/common/validators/strigify-frame-balances.ts new file mode 100644 index 0000000..1c11b37 --- /dev/null +++ b/src/common/validators/strigify-frame-balances.ts @@ -0,0 +1,9 @@ +import { BigNumber } from '@ethersproject/bignumber'; + +export function stringifyFrameBalances(frameBalances: Record) { + return JSON.stringify( + Object.keys(frameBalances).reduce((acc, key) => { + return { ...acc, [key]: frameBalances[key].toString() }; + }, {}), + ); +} diff --git a/src/http/common/middleware/logger.middleware.ts b/src/http/common/middleware/logger.middleware.ts index 270aba4..e203d13 100644 --- a/src/http/common/middleware/logger.middleware.ts +++ b/src/http/common/middleware/logger.middleware.ts @@ -1,7 +1,6 @@ import { Inject, Injectable, LoggerService, NestMiddleware } from '@nestjs/common'; import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston'; import { Request, Reply } from './interfaces'; -import { FastifyRequest } from 'fastify'; @Injectable() export class LoggerMiddleware implements NestMiddleware { @@ -10,15 +9,13 @@ export class LoggerMiddleware implements NestMiddleware { private readonly logger: LoggerService, ) {} - use(request: any, reply: Reply, next: () => void) { + use(request: Request, reply: Reply, next: () => void) { const { ip, method, headers, originalUrl } = request; const userAgent = headers['user-agent'] ?? ''; - const ips = request.ips ? request.ips : []; - reply.on('finish', () => { const { statusCode } = reply; - const log = { method, originalUrl, statusCode, userAgent, ip, ips }; + const log = { method, originalUrl, statusCode, userAgent, ip }; this.logger.log(JSON.stringify(log)); }); diff --git a/src/http/http.constants.ts b/src/http/http.constants.ts index f839bee..b754e63 100644 --- a/src/http/http.constants.ts +++ b/src/http/http.constants.ts @@ -1,6 +1,7 @@ export const HTTP_PATHS = { 1: { nft: 'nft', + 'validators-info': 'validators-info', 'request-time': 'request-time', 'estimate-gas': 'estimate-gas', }, diff --git a/src/http/http.module.ts b/src/http/http.module.ts index fd1d494..0980ffd 100644 --- a/src/http/http.module.ts +++ b/src/http/http.module.ts @@ -12,9 +12,10 @@ import { CacheModule, CacheControlHeadersInterceptor } from './common/cache'; import { RequestTimeModule } from './request-time'; import { NFTModule } from './nft'; import { EstimateModule } from './estimate'; +import { ValidatorsModule } from './validators'; @Module({ - imports: [RequestTimeModule, NFTModule, EstimateModule, CacheModule, ThrottlerModule], + imports: [RequestTimeModule, NFTModule, EstimateModule, ValidatorsModule, CacheModule, ThrottlerModule], providers: [ { provide: APP_GUARD, useClass: ThrottlerBehindProxyGuard }, { provide: APP_INTERCEPTOR, useClass: CacheControlHeadersInterceptor }, diff --git a/src/http/validators/dto/index.ts b/src/http/validators/dto/index.ts new file mode 100644 index 0000000..dfc0484 --- /dev/null +++ b/src/http/validators/dto/index.ts @@ -0,0 +1 @@ +export * from './validators.dto'; diff --git a/src/http/validators/dto/validators.dto.ts b/src/http/validators/dto/validators.dto.ts new file mode 100644 index 0000000..142db42 --- /dev/null +++ b/src/http/validators/dto/validators.dto.ts @@ -0,0 +1,33 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class ValidatorsDto { + @ApiProperty({ + example: 1658650005, + description: 'ms time when data was last updated at', + }) + lastUpdatedAt: number; + + @ApiProperty({ + example: 1724856617, + description: 'max exit epoch over all CL network', + }) + maxExitEpoch: number; + + @ApiProperty({ + example: '{}', + description: 'sum of balances Lido validators with withdrawable_epoch by frame', + }) + frameBalances: Record; + + @ApiProperty({ + example: 100000, + description: 'total number of validators in network', + }) + totalValidators: number; + + @ApiProperty({ + example: 100000, + description: 'current frame', + }) + currentFrame: number; +} diff --git a/src/http/validators/index.ts b/src/http/validators/index.ts new file mode 100644 index 0000000..435ce4b --- /dev/null +++ b/src/http/validators/index.ts @@ -0,0 +1,3 @@ +export * from './validators.controller'; +export * from './validators.module'; +export * from './validators.service'; diff --git a/src/http/validators/validators.controller.ts b/src/http/validators/validators.controller.ts new file mode 100644 index 0000000..40077bd --- /dev/null +++ b/src/http/validators/validators.controller.ts @@ -0,0 +1,28 @@ +import { + ClassSerializerInterceptor, + Controller, + Get, + HttpStatus, + UseInterceptors, + Version, + CacheTTL, +} from '@nestjs/common'; +import { ApiResponse, ApiTags } from '@nestjs/swagger'; +import { HTTP_PATHS } from 'http/http.constants'; +import { ValidatorsService } from './validators.service'; +import { ValidatorsDto } from './dto'; + +@Controller() +@ApiTags('Validators') +@UseInterceptors(ClassSerializerInterceptor) +export class ValidatorsController { + constructor(protected readonly validatorsService: ValidatorsService) {} + + @Version('1') + @Get(HTTP_PATHS[1]['validators-info']) + @CacheTTL(20 * 1000) + @ApiResponse({ status: HttpStatus.OK, type: ValidatorsDto }) + async validatorsV1(): Promise { + return this.validatorsService.getValidatorsInfo(); + } +} diff --git a/src/http/validators/validators.module.ts b/src/http/validators/validators.module.ts new file mode 100644 index 0000000..b772236 --- /dev/null +++ b/src/http/validators/validators.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from 'common/config'; +import { ValidatorsController } from './validators.controller'; +import { ValidatorsService } from './validators.service'; +import { ValidatorsStorageModule } from '../../storage'; +import { GenesisTimeModule } from '../../common/genesis-time'; + +@Module({ + imports: [ConfigModule, ValidatorsStorageModule, GenesisTimeModule], + controllers: [ValidatorsController], + providers: [ValidatorsService], +}) +export class ValidatorsModule {} diff --git a/src/http/validators/validators.service.ts b/src/http/validators/validators.service.ts new file mode 100644 index 0000000..44c7e70 --- /dev/null +++ b/src/http/validators/validators.service.ts @@ -0,0 +1,34 @@ +import { Injectable } from '@nestjs/common'; +import { ConfigService } from 'common/config'; +import { ValidatorsStorageService } from '../../storage'; +import { GenesisTimeService } from '../../common/genesis-time'; + +@Injectable() +export class ValidatorsService { + constructor( + protected readonly configService: ConfigService, + protected readonly validatorsServiceStorage: ValidatorsStorageService, + protected readonly genesisTimeService: GenesisTimeService, + ) {} + + getValidatorsInfo() { + const lastUpdatedAt = this.validatorsServiceStorage.getLastUpdate(); + const maxExitEpoch = Number(this.validatorsServiceStorage.getMaxExitEpoch()); + const frameBalancesBigNumber = this.validatorsServiceStorage.getFrameBalances(); + const totalValidators = this.validatorsServiceStorage.getTotal(); + const currentFrame = this.genesisTimeService.getFrameOfEpoch(this.genesisTimeService.getCurrentEpoch()); + + const frameBalances = Object.keys(frameBalancesBigNumber).reduce((acc, item) => { + acc[item] = frameBalancesBigNumber[item].toString(); + return acc; + }, {} as Record); + + return { + lastUpdatedAt, + maxExitEpoch, + frameBalances, + totalValidators, + currentFrame, + }; + } +} diff --git a/src/jobs/validators/validators.service.ts b/src/jobs/validators/validators.service.ts index b5d48c9..6ed224a 100644 --- a/src/jobs/validators/validators.service.ts +++ b/src/jobs/validators/validators.service.ts @@ -16,6 +16,8 @@ import { ResponseValidatorsData, Validator } from './validators.types'; import { parseGweiToWei } from '../../common/utils/parse-gwei-to-big-number'; import { ValidatorsCacheService } from 'storage/validators/validators-cache.service'; import { CronExpression } from '@nestjs/schedule'; +import { PrometheusService } from '../../common/prometheus'; +import { stringifyFrameBalances } from '../../common/validators/strigify-frame-balances'; export class ValidatorsService { static SERVICE_LOG_NAME = 'validators'; @@ -23,6 +25,7 @@ export class ValidatorsService { constructor( @Inject(LOGGER_PROVIDER) protected readonly logger: LoggerService, + protected readonly prometheusService: PrometheusService, protected readonly consensusProviderService: ConsensusProviderService, protected readonly configService: ConfigService, protected readonly jobService: JobService, @@ -78,23 +81,36 @@ export class ValidatorsService { await unblock(); } - await this.setLidoValidatorsWithdrawableBalances(data); this.validatorsStorageService.setTotal(totalValidators); this.validatorsStorageService.setMaxExitEpoch(latestEpoch); this.validatorsStorageService.setLastUpdate(Math.floor(Date.now() / 1000)); + const frameBalances = await this.getLidoValidatorsWithdrawableBalances(data); + this.validatorsStorageService.setFrameBalances(frameBalances); await this.validatorsCacheService.saveDataToCache(); + const currentFrame = this.genesisTimeService.getFrameOfEpoch(this.genesisTimeService.getCurrentEpoch()); this.logger.log('End update validators', { service: ValidatorsService.SERVICE_LOG_NAME, totalValidators, latestEpoch, + frameBalances: stringifyFrameBalances(frameBalances), + currentFrame, + }); + + Object.keys(frameBalances).forEach((frame) => { + this.prometheusService.validatorsState + .labels({ + frame, + balance: frameBalances[frame], + }) + .inc(); }); }, ); } - protected async setLidoValidatorsWithdrawableBalances(validators: Validator[]) { + protected async getLidoValidatorsWithdrawableBalances(validators: Validator[]) { const keysData = await this.lidoKeys.fetchLidoKeysData(); const lidoValidators = await this.lidoKeys.getLidoValidatorsByKeys(keysData.data, validators); @@ -111,6 +127,6 @@ export class ValidatorsService { await unblock(); } - this.validatorsStorageService.setFrameBalances(frameBalances); + return frameBalances; } } diff --git a/src/storage/validators/validators-cache.service.ts b/src/storage/validators/validators-cache.service.ts index edd0a71..a703ff5 100644 --- a/src/storage/validators/validators-cache.service.ts +++ b/src/storage/validators/validators-cache.service.ts @@ -4,7 +4,7 @@ import * as path from 'path'; import { LOGGER_PROVIDER, LoggerService } from '../../common/logger'; import { ValidatorsStorageService } from './validators.service'; import { BigNumber } from '@ethersproject/bignumber'; -import { parseEther } from '@ethersproject/units'; +import { stringifyFrameBalances } from '../../common/validators/strigify-frame-balances'; @Injectable() export class ValidatorsCacheService { @@ -76,7 +76,7 @@ export class ValidatorsCacheService { this.validatorsStorage.getTotal(), this.validatorsStorage.getMaxExitEpoch(), this.validatorsStorage.getLastUpdate(), - this.stringifyFrameBalances(this.validatorsStorage.getFrameBalances()), + stringifyFrameBalances(this.validatorsStorage.getFrameBalances()), ].join(ValidatorsCacheService.CACHE_DATA_DIVIDER); await writeFile(cacheFileName, data); this.logger.log(`success save to file ${cacheFileName}`, { service: ValidatorsCacheService.SERVICE_LOG_NAME }); @@ -86,18 +86,10 @@ export class ValidatorsCacheService { return path.join(ValidatorsCacheService.CACHE_DIR, ValidatorsCacheService.CACHE_FILE_NAME); }; - protected stringifyFrameBalances(frameBalances: Record) { - return JSON.stringify( - Object.keys(frameBalances).reduce((acc, key) => { - return { ...acc, [key]: frameBalances[key].toString() }; - }, {}), - ); - } - protected parseFrameBalances(frameBalancesStr: string) { const frameBalances = JSON.parse(frameBalancesStr); return Object.keys(frameBalances).reduce((acc, key) => { - return { ...acc, [key]: parseEther(frameBalances[key]) }; + return { ...acc, [key]: BigNumber.from(frameBalances[key]) }; }, {}); } } diff --git a/src/waiting-time/utils/calculate-frame-by-validator-balances.ts b/src/waiting-time/utils/calculate-frame-by-validator-balances.ts index 76bfad0..002fd35 100644 --- a/src/waiting-time/utils/calculate-frame-by-validator-balances.ts +++ b/src/waiting-time/utils/calculate-frame-by-validator-balances.ts @@ -17,7 +17,7 @@ export const calculateFrameByValidatorBalances = (args: calculateFrameByValidato let lastFrame = BigNumber.from(currentFrame); const frames = Object.keys(frameBalances); - let result = null; + let result: BigNumber = null; for (let i = 0; i < frames.length; i++) { const frame = frames[i];