Merge pull request #130 from sotatek-dev/develop

Develop
sotatek-dev · Oct 2, 2024 · eab5062 · eab5062
2 parents 7b8dd54 + 95ce65e
commit eab5062
Show file tree

Hide file tree

Showing 24 changed files with 659 additions and 305 deletions.
diff --git a/.github/workflows/auto-deploy-develop.yml b/.github/workflows/auto-deploy-develop.yml
@@ -15,4 +15,4 @@ jobs:
       - run: |
           echo "${{ vars.MINA_BRIDGE_BE_DEV }}" >> .env
           docker build . -t mina-bridge:1.0.0
-          docker compose -f docker-compose.dev.yaml up -d
+          docker compose -f docker-compose.dev.yaml up -d --remove-orphans
diff --git a/.github/workflows/auto-deploy-test.yml b/.github/workflows/auto-deploy-test.yml
@@ -15,4 +15,4 @@ jobs:
       - run: |
           echo "${{ vars.MINA_BRIDGE_BE_TEST }}" >> .env
           docker build . -t mina-bridge:1.0.0
-          docker compose -f docker-compose.dev.yaml up -d
+          docker compose -f docker-compose.dev.yaml up -d --remove-orphans
diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml
@@ -9,6 +9,7 @@ services:
       - ${PORT}:${PORT}
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
@@ -21,6 +22,7 @@ services:
 
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
@@ -33,6 +35,7 @@ services:
 
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
@@ -45,6 +48,7 @@ services:
 
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
@@ -56,6 +60,7 @@ services:
     restart: always
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
@@ -67,11 +72,13 @@ services:
     restart: always
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
     environment:
       EVM_VALIDATOR_PRIVATE_KEY : ${EVM_VALIDATOR_PRIVATE_KEY_1}
+      THIS_VALIDATOR_INDEX: 1
 
   validate-evm-signature-2:
     image: mina-bridge:1.0.0
@@ -81,11 +88,13 @@ services:
     restart: always
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
     environment:
       EVM_VALIDATOR_PRIVATE_KEY : ${EVM_VALIDATOR_PRIVATE_KEY_2}
+      THIS_VALIDATOR_INDEX: 2
 
   validate-evm-signature-3:
     image: mina-bridge:1.0.0
@@ -95,11 +104,13 @@ services:
     restart: always
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
     environment:
       EVM_VALIDATOR_PRIVATE_KEY : ${EVM_VALIDATOR_PRIVATE_KEY_3}
+      THIS_VALIDATOR_INDEX: 3
   validate-mina-signature-1:
     image: mina-bridge:1.0.0
     command: >
@@ -108,10 +119,12 @@ services:
     restart: always
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     environment:
       MINA_VALIDATOR_PRIVATE_KEY: ${MINA_VALIDATOR_PRIVATE_KEY_1}
+      THIS_VALIDATOR_INDEX: 1
     user: node
   validate-mina-signature-2:
     image: mina-bridge:1.0.0
@@ -121,11 +134,13 @@ services:
     restart: always
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
     environment:
       MINA_VALIDATOR_PRIVATE_KEY: ${MINA_VALIDATOR_PRIVATE_KEY_2}
+      THIS_VALIDATOR_INDEX: 2
   validate-mina-signature-3:
     image: mina-bridge:1.0.0
     command: >
@@ -134,11 +149,25 @@ services:
     restart: always
     depends_on:
       - postgres
+      - redis
     networks:
       - myNetwork
     user: node
     environment:
       MINA_VALIDATOR_PRIVATE_KEY: ${MINA_VALIDATOR_PRIVATE_KEY_3}
+      THIS_VALIDATOR_INDEX: 3
+  job-unlock-provider:
+    image: mina-bridge:1.0.0
+    command: >
+      sh -c "npm run console unlock-job-provider"
+    tty: true
+    restart: always
+    depends_on:
+      - postgres
+      - redis
+    networks:
+      - myNetwork
+    user: node
   postgres:
     container_name: mina-bridge-${NODE_ENV}-postgres
     image: postgres:15.3-alpine3.18
@@ -153,10 +182,34 @@ services:
       POSTGRES_DB: mina-bridge
     networks:
       myNetwork:
-
+  bull_monitor:
+    image: hocptit/bull_ui:0.0.2
+    container_name: mina_bull_ui
+    environment:
+      REDIS_URI: redis://redis:${REDIS_PORT}
+      USERNAME: admin
+      PASSWORD: minabridge
+      QUEUES: UNLOCK_JOB_QUEUE,EVM_SENDER_QUEUE,MINA_SENDER_QUEUE,EVM_VALIDATOR_1,EVM_VALIDATOR_2,EVM_VALIDATOR_2,MINA_VALIDATOR_1,MINA_VALIDATOR_2,MINA_VALIDATOR_3
+    ports:
+      - '${BULL_MONITOR_PORT}:3011'
+    depends_on:
+      - redis
+    networks:
+      - myNetwork
+  redis:
+    image: redis:6.2-alpine
+    container_name: mina_redis
+    ports:
+      - '${REDIS_PORT}:${REDIS_PORT}'
+    command: redis-server --save 20 1 --loglevel warning --port ${REDIS_PORT}
+    networks:
+      - myNetwork
+    volumes:
+      - redisData:/data
 
 volumes:
   postgresData:
+  redisData:
 networks:
   myNetwork:
     name: minaBridgeNetwork${NODE_ENV}
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -14,9 +14,33 @@ services:
       POSTGRES_DB: mina-bridge
     networks:
       myNetwork:
-
+  bull_monitor:
+    image: hocptit/bull_ui:0.0.2
+    container_name: mina_bull_ui
+    environment:
+      REDIS_URI: redis://redis:${REDIS_PORT}
+      USERNAME: admin
+      PASSWORD: minabridge
+      QUEUES: UNLOCK_JOB_QUEUE,EVM_SENDER_QUEUE,MINA_SENDER_QUEUE,EVM_VALIDATOR_1,EVM_VALIDATOR_2,EVM_VALIDATOR_2,MINA_VALIDATOR_1,MINA_VALIDATOR_2,MINA_VALIDATOR_3
+    ports:
+      - '${BULL_MONITOR_PORT}:3011'
+    depends_on:
+      - redis
+    networks:
+      - myNetwork
+  redis:
+    image: redis:6.2-alpine
+    container_name: mina_redis
+    ports:
+      - '${REDIS_PORT}:${REDIS_PORT}'
+    command: redis-server --save 20 1 --loglevel warning --port ${REDIS_PORT}
+    networks:
+      - myNetwork
+    volumes:
+      - redisData:/data
 volumes:
   postgresData:
+  redisData:
 networks:
   myNetwork:
     name: minaBridgeNetwork${NODE_ENV}
diff --git a/src/app.module.ts b/src/app.module.ts
@@ -12,7 +12,9 @@ import { MODULES } from './modules/index.js';
 import { CustomAuthorizationHeaderMiddleware } from './shared/middleware/custom-authorization-header.middleware.js';
 import { LoggerHttpRequestMiddleware } from './shared/middleware/logger-http-request.middleware.js';
 import { LoggingModule } from './shared/modules/logger/logger.module.js';
+import { QueueModule } from './shared/modules/queue/queue.module.js';
 import { Web3Module } from './shared/modules/web3/web3.module.js';
+import { isDevelopmentEnvironment } from './shared/utils/util.js';
 
 const modules = [
   ConfigurationModule,
@@ -21,6 +23,7 @@ const modules = [
   LoggingModule,
   GuardModule,
   Web3Module,
+  QueueModule,
   ConsoleModule,
   ...MODULES,
 ];
@@ -38,7 +41,7 @@ export class AppModule {
       },
     ];
     consumer.apply(CustomAuthorizationHeaderMiddleware).forRoutes(...thirdPartyLoginRoutes);
-    if (process.env.NODE_ENV !== 'production') {
+    if (isDevelopmentEnvironment()) {
       consumer.apply(LoggerHttpRequestMiddleware).forRoutes('*');
     }
   }

diff --git a/src/constants/env.constant.ts b/src/constants/env.constant.ts
@@ -44,9 +44,11 @@ export enum EEnvKey {
   GAS_FEE_EVM = 'GAS_FEE_EVM',
   DECIMAL_TOKEN_EVM = 'DECIMAL_TOKEN_EVM',
   MINA_VALIDATOR_THRESHHOLD = 'MINA_VALIDATOR_THRESHHOLD',
+  EVM_VALIDATOR_THRESHHOLD = 'EVM_VALIDATOR_THRESHHOLD',
   MINA_VALIDATOR_PRIVATE_KEY = 'MINA_VALIDATOR_PRIVATE_KEY',
   EVM_VALIDATOR_PRIVATE_KEY = 'EVM_VALIDATOR_PRIVATE_KEY',
   MINA_CRAWL_SAFE_BLOCK = 'MINA_CRAWL_SAFE_BLOCK',
+  THIS_VALIDATOR_INDEX = 'THIS_VALIDATOR_INDEX',
 }
 
 export enum EEnvironments {

diff --git a/src/constants/error.constant.ts b/src/constants/error.constant.ts
@@ -8,4 +8,5 @@ export enum EError {
   INVALID_SIGNATURE = 'INVALID_SIGNATURE',
   OVER_DAILY_QUOTA = 'OVER_DAILY_QUOTA',
   RESOURCE_NOT_FOUND = 'RESOURCE_NOT_FOUND',
+  NETWORK_FAILED = 'NETWORK_FAILED',
 }
diff --git a/src/constants/queue.constant.ts b/src/constants/queue.constant.ts
@@ -1,2 +1,7 @@
-export const MINA_BRIDGE_CRAWLER_QUEUE = 'MINA_BRIDGE_CRAWLER_QUEUE';
-export const MINA_BRIDGE_SUBMITTER_QUEUE = 'MINA_BRIDGE_SUBMITTER_QUEUE';
+export enum EQueueName {
+  EVM_SENDER_QUEUE = 'EVM_SENDER_QUEUE',
+  MINA_SENDER_QUEUE = 'MINA_SENDER_QUEUE',
+  UNLOCK_JOB_QUEUE = 'UNLOCK_JOB_QUEUE',
+}
+export const getEvmValidatorQueueName = (index: number) => `EVM_VALIDATOR_${index}`;
+export const getMinaValidatorQueueName = (index: number) => `MINA_VALIDATOR_${index}`;
diff --git a/src/database/migrations/1727799403569-unique-validator-tx.ts b/src/database/migrations/1727799403569-unique-validator-tx.ts
@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner, TableUnique } from 'typeorm';
+
+export class UniqueValidatorTx1727799403569 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    return queryRunner.createUniqueConstraint(
+      'multi_signature',
+      new TableUnique({
+        name: 'unique_validator_tx',
+        columnNames: ['validator', 'tx_id'],
+      }),
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    return queryRunner.dropUniqueConstraint('multi_signature','unique_validator_tx')
+  }
+}
diff --git a/src/database/migrations/1727838565058-add-timestamp-check-event-logs.ts b/src/database/migrations/1727838565058-add-timestamp-check-event-logs.ts
@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm';
+
+export class AddTimestampCheckEventLogs1727838565058 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    return queryRunner.addColumns('event_logs', [
+      new TableColumn({ name: 'next_validate_signature_job_time', type: 'bigint', default: 0 }),
+      new TableColumn({ name: 'next_send_tx_job_time', type: 'bigint', default: 0 }),
+    ]);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    return queryRunner.dropColumns('event_logs', ['next_send_tx_job_time', 'next_validate_signature_job_time']);
+  }
+}
diff --git a/src/database/repositories/event-log.repository.ts b/src/database/repositories/event-log.repository.ts
@@ -1,4 +1,3 @@
-import { GetHistoryDto } from 'modules/users/dto/history-response.dto.js';
 import { EntityRepository } from 'nestjs-typeorm-custom-repository';
 import { Brackets } from 'typeorm';
 
@@ -8,7 +7,8 @@ import { ETableName } from '../../constants/entity.constant.js';
 import { MAX_RETRIES } from '../../constants/service.constant.js';
 import { BaseRepository } from '../../core/base-repository.js';
 import { EventLog } from '../../modules/crawler/entities/event-logs.entity.js';
-import { endOfDayUnix, startOfDayUnix } from '../../shared/utils/time.js';
+import { GetHistoryDto } from '../../modules/users/dto/history-response.dto.js';
+import { endOfDayUnix, nowUnix, startOfDayUnix } from '../../shared/utils/time.js';
 
 @EntityRepository(EventLog)
 export class EventLogRepository extends BaseRepository<EventLog> {
@@ -39,13 +39,40 @@ export class EventLogRepository extends BaseRepository<EventLog> {
 
     return qb.getOne();
   }
+  public async getPendingTx(
+    network: ENetworkName,
+    isSignatureFullFilled: boolean,
+    numOfSignaturesNeeded: number,
+  ): Promise<Array<{ id: number; networkReceived: ENetworkName }>> {
+    const currentUnixTimestamp = nowUnix();
+    const qb = this.createQueryBuilder(`${this.alias}`);
+    qb.select([`${this.alias}.id as "id"`, `${this.alias}.network_received as "networkReceived"`]);
+    qb.leftJoin(`${this.alias}.validator`, 'signature');
+
+    qb.where(`${this.alias}.network_received = :network`, { network });
+
+    qb.andWhere(`${this.alias}.status IN (:...status)`, {
+      status: [EEventStatus.WAITING], //  EEventStatus.PROCESSING add in future
+    })
+      .andWhere(`${this.alias}.retry < :retryNumber`, { retryNumber: MAX_RETRIES })
+      .orderBy(`${this.alias}.id`, EDirection.DESC)
+      .groupBy(`${this.alias}.id`)
+      .addGroupBy(`${this.alias}.network_received`);
+    if (isSignatureFullFilled) {
+      qb.andWhere(`${this.alias}.next_send_tx_job_time < :currentUnixTimestamp`, { currentUnixTimestamp });
+      qb.having(`COUNT(signature.id) = :numOfSignaturesNeeded`, { numOfSignaturesNeeded });
+    } else {
+      qb.andWhere(`${this.alias}.next_validate_signature_job_time < :currentUnixTimestamp`, { currentUnixTimestamp });
+      qb.having(`COUNT(signature.id) < :numOfSignaturesNeeded`, { numOfSignaturesNeeded });
+    }
+    return qb.getRawMany();
+  }
 
   public async updateStatusAndRetryEvenLog({
     id,
     ...updateData
   }: {
     id: number;
-    retry: number;
     status: EEventStatus;
     amountReceived?: string;
     protocolFee?: string;