From da0bac6a799be8ab4c68febdb0bb09499033fab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Carneiro?= Date: Fri, 10 Jan 2025 16:59:12 -0500 Subject: [PATCH 1/5] feat: process txs as they arrive --- src/new/wallet.js | 22 +++++++++++-- src/storage/leveldb/store.ts | 4 +++ src/storage/leveldb/utxo_index.ts | 10 ++++++ src/storage/memory_store.ts | 4 +++ src/storage/storage.ts | 15 ++++++++- src/sync/utils.ts | 22 +++++++++++++ src/types.ts | 2 ++ src/utils/storage.ts | 54 +++++++++++++++++++++++++++++++ 8 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 src/sync/utils.ts diff --git a/src/new/wallet.js b/src/new/wallet.js index 1b39e1230..97a046b5b 100644 --- a/src/new/wallet.js +++ b/src/new/wallet.js @@ -51,6 +51,7 @@ import NanoContractTransactionBuilder from '../nano_contracts/builder'; import { prepareNanoSendTransaction } from '../nano_contracts/utils'; import { IHistoryTxSchema } from '../schemas'; import GLL from '../sync/gll'; +import { checkTxMetadataChanged } from '../sync/utils'; /** * @typedef {import('../models/create_token_transaction').default} CreateTokenTransaction @@ -1334,12 +1335,27 @@ class HathorWallet extends EventEmitter { newTx.processingStatus = TxHistoryProcessingStatus.PROCESSING; - // Save the transaction in the storage + // Metadata changed MUST be before addTx because we compare the stored tx with the new one. + // So overwriting the stored tx before this would make the check invalid. + const metadataChanged = await checkTxMetadataChanged(newTx, this.storage); await this.storage.addTx(newTx); - await this.scanAddressesToLoad(); + + // set state to processing and save current state. + const previousState = this.state; + this.state = HathorWallet.PROCESSING; // Process history to update metadatas - await this.storage.processHistory(); + if (metadataChanged) { + // Save the transaction in the storage + await this.storage.processHistory(); + } else { + if (isNewTx) { + // Save the transaction in the storage and process it. + await this.storage.processNewTx(newTx); + } + } + // restore previous state + this.state = previousState; newTx.processingStatus = TxHistoryProcessingStatus.FINISHED; // Save the transaction in the storage diff --git a/src/storage/leveldb/store.ts b/src/storage/leveldb/store.ts index 4c981a08c..6168ae30d 100644 --- a/src/storage/leveldb/store.ts +++ b/src/storage/leveldb/store.ts @@ -268,6 +268,10 @@ export default class LevelDBStore implements IStore { return this.utxoIndex.saveUtxo(utxo); } + async deleteUtxo(utxo: IUtxo): Promise { + await this.utxoIndex.deleteUtxo(utxo); + } + /** * Save a locked utxo to the database. * Used when a new utxo is received but it is either time locked or height locked. diff --git a/src/storage/leveldb/utxo_index.ts b/src/storage/leveldb/utxo_index.ts index 0b603ab5d..f93ed0b74 100644 --- a/src/storage/leveldb/utxo_index.ts +++ b/src/storage/leveldb/utxo_index.ts @@ -306,6 +306,16 @@ export default class LevelUtxoIndex implements IKVUtxoIndex { await this.tokenAddressUtxoDB.put(_token_address_utxo_key(utxo), utxo); await this.tokenUtxoDB.put(_token_utxo_key(utxo), utxo); } + /** + * Remove an utxo from the database. + * @param {IUtxo} utxo utxo to be deleted + * @returns {Promise} + */ + async deleteUtxo(utxo: IUtxo): Promise { + await this.utxoDB.del(_utxo_id(utxo)); + await this.tokenAddressUtxoDB.del(_token_address_utxo_key(utxo)); + await this.tokenUtxoDB.del(_token_utxo_key(utxo)); + } /** * Save a locked utxo on the database. diff --git a/src/storage/memory_store.ts b/src/storage/memory_store.ts index 69e09b2ad..a921319cb 100644 --- a/src/storage/memory_store.ts +++ b/src/storage/memory_store.ts @@ -593,6 +593,10 @@ export class MemoryStore implements IStore { } } + async deleteUtxo(utxo: IUtxo): Promise { + this.utxos.delete(`${utxo.txId}:${utxo.index}`); + } + /** * Fetch utxos based on a selection criteria * @param {IUtxoFilterOptions} options Options to filter utxos diff --git a/src/storage/storage.ts b/src/storage/storage.ts index 57814cb67..3ef4860df 100644 --- a/src/storage/storage.ts +++ b/src/storage/storage.ts @@ -40,7 +40,7 @@ import { getDefaultLogger, } from '../types'; import transactionUtils from '../utils/transaction'; -import { processHistory as processHistoryUtil, processUtxoUnlock } from '../utils/storage'; +import { processHistory as processHistoryUtil, processSingleTx as processSingleTxUtil, processUtxoUnlock } from '../utils/storage'; import config, { Config } from '../config'; import { decryptData, checkPassword } from '../utils/crypto'; import FullNodeConnection from '../new/connection'; @@ -358,6 +358,19 @@ export class Storage implements IStorage { await processHistoryUtil(this, { rewardLock: this.version?.reward_spend_min_blocks }); } + /** + * Process the transaction history to calculate the metadata. + * @returns {Promise} + */ + async processNewTx(tx: IHistoryTx): Promise { + // Add tx to storage + await this.addTx(tx); + // Keep tx-timestamp index sorted + await this.store.preProcessHistory(); + // Process the single tx we received + await processSingleTxUtil(this, tx, { rewardLock: this.version?.reward_spend_min_blocks }); + } + /** * Iterate on all tokens on the storage. * diff --git a/src/sync/utils.ts b/src/sync/utils.ts new file mode 100644 index 000000000..b90830d51 --- /dev/null +++ b/src/sync/utils.ts @@ -0,0 +1,22 @@ +/** + * Copyright (c) Hathor Labs and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import { IHistoryTx, IStorage } from "../types"; + +/** + * Currently we only need to check that a transaction has been voided or un-voided. + */ +export async function checkTxMetadataChanged(tx: IHistoryTx, storage: IStorage): Promise { + const txId = tx.tx_id; + const storageTx = await storage.getTx(txId); + if (!storageTx) { + // This is a new tx + return false; + } + + return tx.is_voided !== storageTx.is_voided; +} diff --git a/src/types.ts b/src/types.ts index fdf750f27..6322e425a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -429,6 +429,7 @@ export interface IStore { saveLockedUtxo(lockedUtxo: ILockedUtxo): Promise; iterateLockedUtxos(): AsyncGenerator; unlockUtxo(lockedUtxo: ILockedUtxo): Promise; + deleteUtxo(utxoId: IUtxo): Promise; // Wallet data getAccessData(): Promise; @@ -502,6 +503,7 @@ export interface IStorage { getSpentTxs(inputs: Input[]): AsyncGenerator<{ tx: IHistoryTx; input: Input; index: number }>; addTx(tx: IHistoryTx): Promise; processHistory(): Promise; + processNewTx(tx: IHistoryTx): Promise; // Tokens isTokenRegistered(tokenUid: string): Promise; diff --git a/src/utils/storage.ts b/src/utils/storage.ts index e3429de42..2846a4d38 100644 --- a/src/utils/storage.ts +++ b/src/utils/storage.ts @@ -22,6 +22,7 @@ import { HistorySyncMode, HistorySyncFunction, WalletType, + IUtxo, } from '../types'; import walletApi from '../api/wallet'; import helpers from './helpers'; @@ -396,6 +397,59 @@ export async function processHistory( await updateWalletMetadataFromProcessedTxData(storage, { maxIndexUsed, tokens }); } +export async function processSingleTx( + storage: IStorage, + tx: IHistoryTx, + { rewardLock }: { rewardLock?: number } = {}, +): Promise { + const { store } = storage; + const nowTs = Math.floor(Date.now() / 1000); + const currentHeight = await store.getCurrentHeight(); + + const tokens = new Set(); + const processedData = await processNewTx(storage, tx, { rewardLock, nowTs, currentHeight }); + const maxIndexUsed = processedData.maxAddressIndex; + for (const token of processedData.tokens) { + tokens.add(token); + } + + for (const input of tx.inputs) { + const origTx = await storage.getTx(input.tx_id); + if (!origTx) { + // The tx being spent is not from the wallet. + continue; + } + if (origTx.outputs.length <= input.index) { + throw new Error('Spending an unexistent output'); + } + const output = origTx.outputs[input.index]; + if (!output.decoded.address) { + // Tx is ours but output is not from an address. + continue; + } + if (!(await storage.isAddressMine(output.decoded.address))) { + // Address is not ours. + continue; + } + const utxo: IUtxo = { + txId: input.tx_id, + index: input.index, + token: output.token, + address: output.decoded.address, + authorities: transactionUtils.isAuthorityOutput(output) ? output.value : 0n, + value: output.value, + timelock: output.decoded?.timelock ?? null, + type: origTx.version, + height: origTx.height ?? null, + }; + // Delete utxo if it is being spent + await store.deleteUtxo(utxo); + } + + // Update wallet data + await updateWalletMetadataFromProcessedTxData(storage, { maxIndexUsed, tokens }); +} + /** * Fetch and save the data of the token set on the storage * @param {IStorage} storage - Storage to save the tokens. From 77c24acb9128012419ac156e6ad61b5b9fb4c0e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Carneiro?= Date: Fri, 10 Jan 2025 17:30:32 -0500 Subject: [PATCH 2/5] chore: linter changes --- src/new/wallet.js | 8 +++----- src/storage/leveldb/utxo_index.ts | 1 + src/storage/storage.ts | 6 +++++- src/sync/utils.ts | 2 +- src/utils/storage.ts | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/new/wallet.js b/src/new/wallet.js index 97a046b5b..a43b50157 100644 --- a/src/new/wallet.js +++ b/src/new/wallet.js @@ -1348,11 +1348,9 @@ class HathorWallet extends EventEmitter { if (metadataChanged) { // Save the transaction in the storage await this.storage.processHistory(); - } else { - if (isNewTx) { - // Save the transaction in the storage and process it. - await this.storage.processNewTx(newTx); - } + } else if (isNewTx) { + // Save the transaction in the storage and process it. + await this.storage.processNewTx(newTx); } // restore previous state this.state = previousState; diff --git a/src/storage/leveldb/utxo_index.ts b/src/storage/leveldb/utxo_index.ts index f93ed0b74..e8a7eb32f 100644 --- a/src/storage/leveldb/utxo_index.ts +++ b/src/storage/leveldb/utxo_index.ts @@ -306,6 +306,7 @@ export default class LevelUtxoIndex implements IKVUtxoIndex { await this.tokenAddressUtxoDB.put(_token_address_utxo_key(utxo), utxo); await this.tokenUtxoDB.put(_token_utxo_key(utxo), utxo); } + /** * Remove an utxo from the database. * @param {IUtxo} utxo utxo to be deleted diff --git a/src/storage/storage.ts b/src/storage/storage.ts index 3ef4860df..44dfdca36 100644 --- a/src/storage/storage.ts +++ b/src/storage/storage.ts @@ -40,7 +40,11 @@ import { getDefaultLogger, } from '../types'; import transactionUtils from '../utils/transaction'; -import { processHistory as processHistoryUtil, processSingleTx as processSingleTxUtil, processUtxoUnlock } from '../utils/storage'; +import { + processHistory as processHistoryUtil, + processSingleTx as processSingleTxUtil, + processUtxoUnlock, +} from '../utils/storage'; import config, { Config } from '../config'; import { decryptData, checkPassword } from '../utils/crypto'; import FullNodeConnection from '../new/connection'; diff --git a/src/sync/utils.ts b/src/sync/utils.ts index b90830d51..3483241bc 100644 --- a/src/sync/utils.ts +++ b/src/sync/utils.ts @@ -5,7 +5,7 @@ * LICENSE file in the root directory of this source tree. */ -import { IHistoryTx, IStorage } from "../types"; +import { IHistoryTx, IStorage } from '../types'; /** * Currently we only need to check that a transaction has been voided or un-voided. diff --git a/src/utils/storage.ts b/src/utils/storage.ts index 2846a4d38..8b723dc68 100644 --- a/src/utils/storage.ts +++ b/src/utils/storage.ts @@ -400,7 +400,7 @@ export async function processHistory( export async function processSingleTx( storage: IStorage, tx: IHistoryTx, - { rewardLock }: { rewardLock?: number } = {}, + { rewardLock }: { rewardLock?: number } = {} ): Promise { const { store } = storage; const nowTs = Math.floor(Date.now() / 1000); From 49d3ae96373117664c570386ea8bd69958fc0475 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 14 Jan 2025 11:49:45 -0300 Subject: [PATCH 3/5] feat: use cloneDeep to get token data from storage in the get balance --- src/new/wallet.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/new/wallet.js b/src/new/wallet.js index a43b50157..ff86487b4 100644 --- a/src/new/wallet.js +++ b/src/new/wallet.js @@ -5,7 +5,7 @@ * LICENSE file in the root directory of this source tree. */ -import { get } from 'lodash'; +import { cloneDeep, get } from 'lodash'; import bitcore, { HDPrivateKey } from 'bitcore-lib'; import EventEmitter from 'events'; import { NATIVE_TOKEN_UID, P2SH_ACCT_PATH, P2PKH_ACCT_PATH } from '../constants'; @@ -699,7 +699,9 @@ class HathorWallet extends EventEmitter { throw new WalletError('Not implemented.'); } const uid = token || this.token.uid; - let tokenData = await this.storage.getToken(uid); + // Using clone deep so the balance returned will not be updated in case + // we change the storage + let tokenData = cloneDeep(await this.storage.getToken(uid)); if (tokenData === null) { // We don't have the token on storage, so we need to return an empty default response tokenData = { From b822a044d50a517458fb43dca57bae4297f229df Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 14 Jan 2025 16:28:04 -0300 Subject: [PATCH 4/5] refactor: remove uneeded method call and improve methods comments --- src/new/wallet.js | 9 +++++++-- src/storage/storage.ts | 2 -- src/utils/storage.ts | 10 ++++++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/new/wallet.js b/src/new/wallet.js index ff86487b4..d5881b3e5 100644 --- a/src/new/wallet.js +++ b/src/new/wallet.js @@ -1348,10 +1348,15 @@ class HathorWallet extends EventEmitter { this.state = HathorWallet.PROCESSING; // Process history to update metadatas if (metadataChanged) { - // Save the transaction in the storage + // Process the full history because + // this tx changed the voided state + // XXX in the future we should be able to handle this + // voidness alone, without processing everything + // but for now it's an isolated case and it's easier await this.storage.processHistory(); } else if (isNewTx) { - // Save the transaction in the storage and process it. + // Process this single transaction. + // Handling new metadatas and deleting utxos that are not available anymore await this.storage.processNewTx(newTx); } // restore previous state diff --git a/src/storage/storage.ts b/src/storage/storage.ts index 44dfdca36..e88b75232 100644 --- a/src/storage/storage.ts +++ b/src/storage/storage.ts @@ -367,8 +367,6 @@ export class Storage implements IStorage { * @returns {Promise} */ async processNewTx(tx: IHistoryTx): Promise { - // Add tx to storage - await this.addTx(tx); // Keep tx-timestamp index sorted await this.store.preProcessHistory(); // Process the single tx we received diff --git a/src/utils/storage.ts b/src/utils/storage.ts index 8b723dc68..1d9afdc90 100644 --- a/src/utils/storage.ts +++ b/src/utils/storage.ts @@ -419,18 +419,23 @@ export async function processSingleTx( // The tx being spent is not from the wallet. continue; } + if (origTx.outputs.length <= input.index) { throw new Error('Spending an unexistent output'); } + const output = origTx.outputs[input.index]; if (!output.decoded.address) { // Tx is ours but output is not from an address. continue; } + if (!(await storage.isAddressMine(output.decoded.address))) { // Address is not ours. continue; } + + // Now we get the utxo object to be deleted from the store const utxo: IUtxo = { txId: input.tx_id, index: input.index, @@ -442,11 +447,12 @@ export async function processSingleTx( type: origTx.version, height: origTx.height ?? null, }; - // Delete utxo if it is being spent + + // Delete utxo await store.deleteUtxo(utxo); } - // Update wallet data + // Update wallet data in the store await updateWalletMetadataFromProcessedTxData(storage, { maxIndexUsed, tokens }); } From 54c51ceead6667c3c29427d793bcfb71f71dd62f Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 22 Jan 2025 16:33:49 -0300 Subject: [PATCH 5/5] tests: add test to validate the full processHistory call on voidness --- .../integration/nanocontracts/bet.test.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/__tests__/integration/nanocontracts/bet.test.ts b/__tests__/integration/nanocontracts/bet.test.ts index 01afd2919..1c8635531 100644 --- a/__tests__/integration/nanocontracts/bet.test.ts +++ b/__tests__/integration/nanocontracts/bet.test.ts @@ -4,6 +4,7 @@ import { generateMultisigWalletHelper, generateWalletHelper, waitForTxReceived, + waitNextBlock, waitTxConfirmed, } from '../helpers/wallet.helper'; import { NATIVE_TOKEN_UID, NANO_CONTRACTS_INITIALIZE_METHOD } from '../../../src/constants'; @@ -455,6 +456,29 @@ describe('full cycle of bet nano contract', () => { 200 ); expect(ncStateFirstBlockHeight.fields[`withdrawals.a'${address3}'`].value).toBeUndefined(); + + // Test a tx that becomes voided after the nano execution + const txWithdrawal2 = await wallet.createAndSendNanoContractTransaction('withdraw', address2, { + ncId: tx1.hash, + actions: [ + { + type: 'withdrawal', + token: NATIVE_TOKEN_UID, + amount: 400n, + address: address2, + }, + ], + }); + + jest.spyOn(wallet.storage, 'processHistory'); + expect(wallet.storage.processHistory.mock.calls.length).toBe(0); + await waitNextBlock(wallet.storage); + const txWithdrawal2Data = await wallet.getFullTxById(txWithdrawal2.hash); + + // The tx became voided after the block because of the nano execution + // This voidness called the full processHistory method + expect(isEmpty(txWithdrawal2Data.meta.voided_by)).toBe(false); + expect(wallet.storage.processHistory.mock.calls.length).toBe(1); }; it('bet deposit', async () => {