Merge pull request #253 from lidofinance/develop

Develop to main

how-estimation-works.md

@@ -85,14 +85,21 @@ where `unfinalized` is the amount of the withdrawal request considered summed wi
 
 If there is not enough ether to fulfill the withdrawal request (`unfinalized > totalBuffer`), the previous case might be appended with the known validators are to be withdrawn (when the `withdrawable_epoch` is assigned).
 
-It's needed to select the Lido-participating validators which are already in process of withdrawal and group them by `withdrawable_epoch` to `frameBalances`, allowing to find the oracle report frame containing enough funds from:
+It's needed to select the Lido-participating validators which are already in process of withdrawal and group them by calculated frame of expected withdrawal to `frameBalances`, allowing to find the oracle report frame containing enough funds from:
 
 - buffer (`totalBuffer`)
 - projectedRewards (`rewardsPerEpoch * epochsTillTheFrame`)
-- frameBalances (`object { [frame]: [sum of balances of validators with withdrawable_epoch for certain frame] }`)
+- frameBalances (`object { [frame]: [sum of balances of validators with calculated withdrawal frame] }`)
 
-So the final formula for that case looks like this:
-`frame (which has engough validator balances) + sweepingMean`. More about `sweepingMean` [here](#sweeping mean).
+#### Algorithm of calculation withdrawal frame of validators:
+
+1. Withdrawals sweep cursor goes from 0 to the last validator index in infinite loop.
+2. When the cursor reaches a withdrawable validator, it withdraws ETH from that validator.
+3. The cursor can withdraw from a maximum of 16 validators per slot.
+4. We assume that all validators in network have to something to withdraw (partially or fully)
+5. `percentOfActiveValidators` is used to exclude inactive validators from the queue, ensuring more accurate calculations.
+6. Formula to get number of slots to wait is `(number of validators to withdraw before cursor get index of validator) / 16`
+7. By knowing number slots we can calculate frame of withdrawal 
 
 ---
 

package.json

@@ -27,7 +27,7 @@
   "dependencies": {
     "@ethersproject/bignumber": "^5.7.0",
     "@ethersproject/units": "^5.7.0",
-    "@fastify/static": "^6.6.0",
+    "@fastify/static": "^7.0.4",
     "@lido-nestjs/consensus": "^1.5.0",
     "@lido-nestjs/constants": "^5.2.0",
     "@lido-nestjs/contracts": "^9.3.0",
@@ -37,12 +37,12 @@
     "@lido-nestjs/logger": "^1.0.1",
     "@lidofinance/satanizer": "^0.32.0",
     "@nestjs/cache-manager": "^2.2.2",
-    "@nestjs/common": "^9.2.1",
+    "@nestjs/common": "^10.4.4",
     "@nestjs/config": "^2.2.0",
-    "@nestjs/core": "^9.2.1",
-    "@nestjs/platform-fastify": "^9.2.1",
+    "@nestjs/core": "^10.4.4",
+    "@nestjs/platform-fastify": "^10.4.4",
     "@nestjs/schedule": "^2.2.0",
-    "@nestjs/swagger": "^6.1.4",
+    "@nestjs/swagger": "^7.4.2",
     "@nestjs/terminus": "^9.1.4",
     "@nestjs/throttler": "^6.0.0",
     "@sentry/node": "^7.29.0",
@@ -51,7 +51,6 @@
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",
     "ethers": "^6.13.2",
-    "fastify-swagger": "^5.2.0",
     "node-abort-controller": "^3.0.1",
     "prom-client": "^14.1.1",
     "reflect-metadata": "^0.1.13",
@@ -62,8 +61,8 @@
   },
   "devDependencies": {
     "@nestjs/cli": "^10.4.4",
-    "@nestjs/schematics": "^9.0.4",
-    "@nestjs/testing": "^9.2.1",
+    "@nestjs/schematics": "^10.1.4",
+    "@nestjs/testing": "^10.4.4",
     "@types/cron": "^2.0.1",
     "@types/jest": "^29.2.5",
     "@types/node": "^18.15.11",
@@ -80,7 +79,7 @@
     "ts-loader": "^9.2.6",
     "ts-node": "^10.4.0",
     "tsconfig-paths": "^4.1.2",
-    "typescript": "^4.5.4"
+    "typescript": "^5.1.0"
   },
   "jest": {
     "moduleFileExtensions": [

src/common/execution-provider/execution-provider.service.ts

@@ -1,10 +1,17 @@
 import { SimpleFallbackJsonRpcBatchProvider } from '@lido-nestjs/execution';
 import { CHAINS } from '@lido-nestjs/constants';
 import { Injectable } from '@nestjs/common';
+import { ethers } from 'ethers';
+import { ConfigService } from '@nestjs/config';
+import { PrometheusService } from '../prometheus';
 
 @Injectable()
 export class ExecutionProviderService {
-  constructor(protected readonly provider: SimpleFallbackJsonRpcBatchProvider) {}
+  constructor(
+    protected readonly provider: SimpleFallbackJsonRpcBatchProvider,
+    protected readonly prometheusService: PrometheusService,
+    protected readonly configService: ConfigService,
+  ) {}
 
   /**
    * Returns network name
@@ -22,4 +29,19 @@ export class ExecutionProviderService {
     const { chainId } = await this.provider.getNetwork();
     return chainId;
   }
+
+  // using ethers.JsonRpcProvider direct request to "eth_getBlockByNumber"
+  // default @ethersproject provider getBlock does not contain "withdrawals" property
+  public async getLatestWithdrawals(): Promise<Array<{ validatorIndex: string }>> {
+    const endTimer = this.prometheusService.elRpcRequestDuration.startTimer();
+    try {
+      const provider = new ethers.JsonRpcProvider(this.configService.get('EL_RPC_URLS')[0]);
+      const block = await provider.send('eth_getBlockByNumber', ['latest', false]);
+      endTimer({ result: 'success' });
+      return block.withdrawals;
+    } catch (error) {
+      endTimer({ result: 'error' });
+      throw error;
+    }
+  }
 }

src/http/estimate/estimate.controller.ts

@@ -5,9 +5,9 @@ import {
   HttpStatus,
   UseInterceptors,
   Version,
-  CacheTTL,
   Query,
 } from '@nestjs/common';
+import { CacheTTL } from '@nestjs/cache-manager';
 import { ApiResponse, ApiTags } from '@nestjs/swagger';
 import { Throttle } from '@nestjs/throttler';
 import { HTTP_PATHS } from 'http/http.constants';

src/http/nft/nft.controller.ts

@@ -5,11 +5,11 @@ import {
   HttpStatus,
   UseInterceptors,
   Version,
-  CacheTTL,
   Param,
   Query,
   Header,
 } from '@nestjs/common';
+import { CacheTTL } from '@nestjs/cache-manager';
 import { ApiResponse, ApiTags } from '@nestjs/swagger';
 import { Throttle } from '@nestjs/throttler';
 import { HTTP_PATHS } from 'http/http.constants';

src/http/validators/validators.controller.ts

@@ -1,12 +1,5 @@
-import {
-  ClassSerializerInterceptor,
-  Controller,
-  Get,
-  HttpStatus,
-  UseInterceptors,
-  Version,
-  CacheTTL,
-} from '@nestjs/common';
+import { ClassSerializerInterceptor, Controller, Get, HttpStatus, UseInterceptors, Version } from '@nestjs/common';
+import { CacheTTL } from '@nestjs/cache-manager';
 import { ApiResponse, ApiTags } from '@nestjs/swagger';
 import { HTTP_PATHS } from 'http/http.constants';
 import { ValidatorsService } from './validators.service';

src/http/validators/validators.service.ts

@@ -15,7 +15,7 @@ export class ValidatorsService {
     const lastUpdatedAt = this.validatorsServiceStorage.getLastUpdate();
     const maxExitEpoch = Number(this.validatorsServiceStorage.getMaxExitEpoch());
     const frameBalancesBigNumber = this.validatorsServiceStorage.getFrameBalances();
-    const totalValidators = this.validatorsServiceStorage.getTotal();
+    const totalValidators = this.validatorsServiceStorage.getActiveValidatorsCount();
     const currentFrame = this.genesisTimeService.getFrameOfEpoch(this.genesisTimeService.getCurrentEpoch());
 
     const frameBalances = Object.keys(frameBalancesBigNumber).reduce((acc, item) => {

src/jobs/validators/utils/get-validator-withdrawal-timestamp.ts

@@ -0,0 +1,44 @@
+import { BigNumber } from '@ethersproject/bignumber';
+import { WITHDRAWALS_VALIDATORS_PER_SLOT } from '../validators.constants';
+import { SECONDS_PER_SLOT } from 'common/genesis-time';
+
+/*
+#### Algorithm of calculation withdrawal frame of validators:
+
+1. Withdrawals sweep cursor goes from 0 to the last validator index in infinite loop.
+2. When the cursor reaches a withdrawable validator, it withdraws ETH from that validator.
+3. The cursor can withdraw from a maximum of 16 validators per slot.
+4. We assume that all validators in network have to something to withdraw (partially or fully)
+5. `percentOfActiveValidators` is used to exclude inactive validators from the queue, ensuring more accurate calculations.
+6. Formula to get number of slots to wait is `(number of validators to withdraw before cursor get index of validator) / 16`
+7. By knowing number slots we can calculate frame of withdrawal
+
+Examples:
+
+1. If the current cursor is 50 and the total number of validators is 100,
+   then if we want to know when the validator with index 75 will be withdrawn:
+   (75 - 50) / 16 = 2 slots.
+
+2. If the current cursor is 50 and the total number of validators is 100,
+   and we want to know when the validator with index 25 will be withdrawn
+   (since the cursor will go to the end and start from 0):
+   (100 - 50 + 25) / 16 = 5 slots.
+
+*/
+export function getValidatorWithdrawalTimestamp(
+  index: BigNumber,
+  lastWithdrawalValidatorIndex: BigNumber,
+  activeValidatorCount: number,
+  totalValidatorsCount: number,
+) {
+  const diff = index.sub(lastWithdrawalValidatorIndex);
+  const percentOfActiveValidators = activeValidatorCount / totalValidatorsCount;
+  const lengthQueueValidators = diff.lt(0)
+    ? BigNumber.from(activeValidatorCount).sub(lastWithdrawalValidatorIndex).add(index)
+    : diff;
+
+  const slots = lengthQueueValidators.div(BigNumber.from(WITHDRAWALS_VALIDATORS_PER_SLOT));
+  const seconds = slots.toNumber() * SECONDS_PER_SLOT * percentOfActiveValidators;
+
+  return Date.now() + seconds * 1000;
+}

src/jobs/validators/validators.constants.ts

@@ -9,3 +9,5 @@ export const ORACLE_REPORTS_CRON_BY_CHAIN_ID = {
   [CHAINS.Mainnet]: '30 4/8 * * *', // 4 utc, 12 utc, 20 utc
   [CHAINS.Holesky]: CronExpression.EVERY_3_HOURS, // happens very often, not necessary sync in testnet
 };
+
+export const WITHDRAWALS_VALIDATORS_PER_SLOT = 16;

src/jobs/validators/validators.service.ts

@@ -4,20 +4,22 @@ import { LOGGER_PROVIDER, LoggerService } from 'common/logger';
 import { JobService } from 'common/job';
 import { ConfigService } from 'common/config';
 import { ConsensusProviderService } from 'common/consensus-provider';
+import { ExecutionProviderService } from 'common/execution-provider';
 import { GenesisTimeService } from 'common/genesis-time';
 import { OneAtTime } from '@lido-nestjs/decorators';
 import { ValidatorsStorageService } from 'storage';
 import { FAR_FUTURE_EPOCH, ORACLE_REPORTS_CRON_BY_CHAIN_ID, MAX_SEED_LOOKAHEAD } from './validators.constants';
 import { BigNumber } from '@ethersproject/bignumber';
 import { processValidatorsStream } from 'jobs/validators/utils/validators-stream';
-import { unblock } from '../../common/utils/unblock';
+import { unblock } from 'common/utils/unblock';
 import { LidoKeysService } from './lido-keys';
 import { ResponseValidatorsData, Validator } from './validators.types';
-import { parseGweiToWei } from '../../common/utils/parse-gwei-to-big-number';
+import { parseGweiToWei } from 'common/utils/parse-gwei-to-big-number';
 import { ValidatorsCacheService } from 'storage/validators/validators-cache.service';
 import { CronExpression } from '@nestjs/schedule';
-import { PrometheusService } from '../../common/prometheus';
-import { stringifyFrameBalances } from '../../common/validators/strigify-frame-balances';
+import { PrometheusService } from 'common/prometheus';
+import { stringifyFrameBalances } from 'common/validators/strigify-frame-balances';
+import { getValidatorWithdrawalTimestamp } from './utils/get-validator-withdrawal-timestamp';
 
 export class ValidatorsService {
   static SERVICE_LOG_NAME = 'validators';
@@ -27,6 +29,7 @@ export class ValidatorsService {
 
     protected readonly prometheusService: PrometheusService,
     protected readonly consensusProviderService: ConsensusProviderService,
+    protected readonly executionProviderService: ExecutionProviderService,
     protected readonly configService: ConfigService,
     protected readonly jobService: JobService,
     protected readonly validatorsStorageService: ValidatorsStorageService,
@@ -65,12 +68,12 @@ export class ValidatorsService {
         const data: ResponseValidatorsData = await processValidatorsStream(stream);
         const currentEpoch = this.genesisTimeService.getCurrentEpoch();
 
-        let totalValidators = 0;
+        let activeValidatorCount = 0;
         let latestEpoch = `${currentEpoch + MAX_SEED_LOOKAHEAD + 1}`;
 
         for (const item of data) {
           if (['active_ongoing', 'active_exiting', 'active_slashed'].includes(item.status)) {
-            totalValidators++;
+            activeValidatorCount++;
           }
 
           if (item.validator.exit_epoch !== FAR_FUTURE_EPOCH.toString()) {
@@ -81,7 +84,9 @@ export class ValidatorsService {
 
           await unblock();
         }
-        this.validatorsStorageService.setTotal(totalValidators);
+
+        this.validatorsStorageService.setActiveValidatorsCount(activeValidatorCount);
+        this.validatorsStorageService.setTotalValidatorsCount(data.length);
         this.validatorsStorageService.setMaxExitEpoch(latestEpoch);
         this.validatorsStorageService.setLastUpdate(Math.floor(Date.now() / 1000));
 
@@ -92,7 +97,7 @@ export class ValidatorsService {
         const currentFrame = this.genesisTimeService.getFrameOfEpoch(this.genesisTimeService.getCurrentEpoch());
         this.logger.log('End update validators', {
           service: ValidatorsService.SERVICE_LOG_NAME,
-          totalValidators,
+          activeValidatorCount,
           latestEpoch,
           frameBalances: stringifyFrameBalances(frameBalances),
           currentFrame,
@@ -113,12 +118,18 @@ export class ValidatorsService {
   protected async getLidoValidatorsWithdrawableBalances(validators: Validator[]) {
     const keysData = await this.lidoKeys.fetchLidoKeysData();
     const lidoValidators = await this.lidoKeys.getLidoValidatorsByKeys(keysData.data, validators);
-
+    const lastWithdrawalValidatorIndex = await this.getLastWithdrawalValidatorIndex();
     const frameBalances = {};
 
     for (const item of lidoValidators) {
       if (item.validator.withdrawable_epoch !== FAR_FUTURE_EPOCH.toString() && BigNumber.from(item.balance).gt(0)) {
-        const frame = this.genesisTimeService.getFrameOfEpoch(Number(item.validator.withdrawable_epoch));
+        const withdrawalTimestamp = getValidatorWithdrawalTimestamp(
+          BigNumber.from(item.index),
+          lastWithdrawalValidatorIndex,
+          this.validatorsStorageService.getActiveValidatorsCount(),
+          this.validatorsStorageService.getTotalValidatorsCount(),
+        );
+        const frame = this.genesisTimeService.getFrameByTimestamp(withdrawalTimestamp) + 1;
         const prevBalance = frameBalances[frame];
         const balance = parseGweiToWei(item.balance);
         frameBalances[frame] = prevBalance ? prevBalance.add(balance) : BigNumber.from(balance);
@@ -129,4 +140,9 @@ export class ValidatorsService {
 
     return frameBalances;
   }
+
+  protected async getLastWithdrawalValidatorIndex() {
+    const withdrawals = await this.executionProviderService.getLatestWithdrawals();
+    return BigNumber.from(withdrawals[withdrawals.length - 1].validatorIndex);
+  }
 }

src/storage/validators/validators-cache.service.ts

@@ -50,7 +50,7 @@ export class ValidatorsCacheService {
         return;
       }
 
-      this.validatorsStorage.setTotal(Number(data[0]));
+      this.validatorsStorage.setActiveValidatorsCount(Number(data[0]));
       this.validatorsStorage.setMaxExitEpoch(data[1]);
       this.validatorsStorage.setLastUpdate(Number(data[2]));
       this.validatorsStorage.setFrameBalances(this.parseFrameBalances(data[3]));
@@ -73,7 +73,7 @@ export class ValidatorsCacheService {
 
     await mkdir(ValidatorsCacheService.CACHE_DIR, { recursive: true });
     const data = [
-      this.validatorsStorage.getTotal(),
+      this.validatorsStorage.getActiveValidatorsCount(),
       this.validatorsStorage.getMaxExitEpoch(),
       this.validatorsStorage.getLastUpdate(),
       stringifyFrameBalances(this.validatorsStorage.getFrameBalances()),

src/storage/validators/validators.service.ts

@@ -4,7 +4,8 @@ import { BigNumber } from '@ethersproject/bignumber';
 @Injectable()
 export class ValidatorsStorageService {
   protected maxExitEpoch: string;
-  protected total: number;
+  protected activeValidatorsCount: number;
+  protected totalValidatorsCount: number;
   protected lastUpdate: number;
   protected frameBalances: Record<string, BigNumber>;
 
@@ -20,8 +21,8 @@ export class ValidatorsStorageService {
    * Get total validators
    * @returns total validators number
    */
-  public getTotal(): number {
-    return this.total;
+  public getActiveValidatorsCount(): number {
+    return this.activeValidatorsCount;
   }
 
   /**
@@ -42,10 +43,10 @@ export class ValidatorsStorageService {
 
   /**
    * Updates total validators
-   * @param total - total validators number
+   * @param activeValidatorsCount - total validators number
    */
-  public setTotal(total: number): void {
-    this.total = total;
+  public setActiveValidatorsCount(activeValidatorsCount: number): void {
+    this.activeValidatorsCount = activeValidatorsCount;
   }
 
   /**
@@ -71,4 +72,12 @@ export class ValidatorsStorageService {
   public setFrameBalances(frameBalances: Record<string, BigNumber>): void {
     this.frameBalances = frameBalances;
   }
+
+  public setTotalValidatorsCount(totalValidatorsCount: number) {
+    this.totalValidatorsCount = totalValidatorsCount;
+  }
+
+  public getTotalValidatorsCount() {
+    return this.totalValidatorsCount;
+  }
 }