diff --git a/.env b/.env index 0464817b..051e70e0 100644 --- a/.env +++ b/.env @@ -16,7 +16,7 @@ PLATFORM_JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc*:gc.lo NETWORK_NODE_LOGS_ROOT_PATH=./network-logs/node APPLICATION_ROOT_PATH=./compose-network/network-node APPLICATION_CONFIG_PATH=./compose-network/network-node/data/config -RECORD_PARSER_ROOT_PATH=./src/record-parser +RECORD_PARSER_ROOT_PATH=./src/services/record-parser #### Network Node Memory Limits #### NETWORK_NODE_MEM_LIMIT=8gb @@ -78,6 +78,7 @@ RELAY_RATE_LIMIT_DISABLED=true #### Record Stream Uploader #### STREAM_EXTENSION=rcd.gz +STREAM_SIG_EXTENSION=rcd_sig #### ENVOY #### ENVOY_IMAGE_PREFIX=envoyproxy/ diff --git a/README.md b/README.md index eaaef733..1c0a1712 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ Available commands: --balance to set starting hbar balance of the created accounts. --async to enable or disable asynchronous creation of accounts. --b or --blocklist to enable or disable account blocklisting. Depending on how many private keys are blocklisted, this will affect the generated on startup accounts. + --enable-debug Enable or disable debugging of the local node [boolean] [default: false] stop - Stops the local hedera network and delete all the existing data. restart - Restart the local hedera network. generate-accounts - Generates N accounts, default 10. @@ -116,6 +117,9 @@ Available commands: --h or --host to override the default host. --balance to set starting hbar balance of the created accounts. --async to enable or disable asynchronous creation of accounts. + debug [timestamp] - Parses and prints the contents of the record file that has been created + during the selected timestamp. + Important: Local node must be started with the -g, --enable-debug flag to enable this feature ``` Note: Generated accounts are 3 types (ECDSA, Alias ECDSA and ED25519). All of them are usable via HederaSDK. Only Alias ECDSA accounts can be imported into wallet like Metamask or used in ethers. diff --git a/src/Errors/LocalNodeErrors.ts b/src/Errors/LocalNodeErrors.ts index 918b5232..75bd18a9 100644 --- a/src/Errors/LocalNodeErrors.ts +++ b/src/Errors/LocalNodeErrors.ts @@ -18,7 +18,7 @@ * */ -export class LocalNodeErrors extends Error{ +export class LocalNodeErrors extends Error { public message: string; public name: string; @@ -39,5 +39,8 @@ export class LocalNodeErrors extends Error{ export const Errors = { CONNECTION_ERROR: (port?: number) => new LocalNodeErrors("Connection Error", `Something went wrong, while trying to connect ${port ? `to port ${port}` : `to local node`}`), - CLEINT_ERROR: (msg?: string) => new LocalNodeErrors("Client Error", `Something went wrong, while trying to create SDK Client${msg ? `: ${msg}` : ``}`) + CLEINT_ERROR: (msg?: string) => new LocalNodeErrors("Client Error", `Something went wrong, while trying to create SDK Client${msg ? `: ${msg}` : ``}`), + NO_RECORD_FILE_FOUND_ERROR: () => new LocalNodeErrors('No record file found Error', "This record file doesn't not exist, check if timestamp is correct and local-node was started in debug mode using --enable-debug option"), + INVALID_TIMESTAMP_ERROR: () => new LocalNodeErrors('Invalid Timestamp Error', 'Invalid timestamp string. Accepted formats are: 0000000000.000000000 and 0000000000-000000000'), + DEBUG_MODE_CHECK_ERROR: () => new LocalNodeErrors('Debug Mode check Error', 'Debug mode is not enabled to use this command. Please use the --enable-debug flag to enable it.'), } diff --git a/src/configuration/originalNodeConfiguration.json b/src/configuration/originalNodeConfiguration.json index 63b2a2f1..0e7c0df6 100644 --- a/src/configuration/originalNodeConfiguration.json +++ b/src/configuration/originalNodeConfiguration.json @@ -33,5 +33,8 @@ "accountId": "0.0.6", "host": "network-node-3" } - ] + ], + "local": { + "deleteAfterProcessing": false + } } diff --git a/src/constants.ts b/src/constants.ts index 2f24a4f5..90836143 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -41,6 +41,9 @@ export const MIRROR_NODE_LABEL = "mirror-node-rest"; export const RELAY_LABEL = "json-rpc-relay"; export const IS_WINDOWS = process.platform === "win32"; export const UNKNOWN_VERSION = "Unknown"; -export const EVM_ADDRESSES_BLOCKLIST_FILE_RELATIVE_PATH = '../../compose-network/network-node'; export const NECESSARY_PORTS = [5551, 8545, 5600, 5433, 50211, 8082]; export const OPTIONAL_PORTS = [7546, 8080, 6379, 3000]; +export const EVM_ADDRESSES_BLOCKLIST_FILE_RELATIVE_PATH = '../../compose-network/network-node' +export const RELATIVE_TMP_DIR_PATH = '../../src/services/record-parser/temp'; +export const RELATIVE_RECORDS_DIR_PATH = '../../network-logs/node/recordStreams/record0.0.3'; +export const APPLICATION_YML_RELATIVE_PATH = '../../compose-network/mirror-node/application.yml'; diff --git a/src/data/StateData.ts b/src/data/StateData.ts index 0dd21c20..dd865d0a 100644 --- a/src/data/StateData.ts +++ b/src/data/StateData.ts @@ -26,6 +26,7 @@ import { NetworkPrepState } from '../state/NetworkPrepState'; import { StartState } from '../state/StartState'; import { StopState } from '../state/StopState'; import { StateConfiguration } from '../types/StateConfiguration'; +import { DebugState } from '../state/DebugState'; export class StateData { @@ -39,6 +40,8 @@ export class StateData { return this.getStopConfiguration(); case 'accountCreation': return this.getAccountCreationConfiguration(); + case 'debug': + return this.getDebugConfiguration(); default: return undefined; } @@ -92,4 +95,13 @@ export class StateData { ] } } + + private getDebugConfiguration(): StateConfiguration { + return { + 'stateMachineName' : 'debug', + 'states' : [ + new DebugState() + ] + } + } } diff --git a/src/services/CLIService.ts b/src/services/CLIService.ts index 3a51ba7e..58822eb8 100644 --- a/src/services/CLIService.ts +++ b/src/services/CLIService.ts @@ -57,6 +57,7 @@ export class CLIService implements IService{ CLIService.userComposeOption(yargs); CLIService.userComposeDirOption(yargs); CLIService.blocklistingOption(yargs); + CLIService.enableDebugOption(yargs); } public static loadDebugOptions(yargs: Argv<{}>): void { @@ -97,6 +98,8 @@ export class CLIService implements IService{ const blocklisting = argv.blocklist as boolean; const startup = argv.startup as boolean; const verbose = CLIService.resolveVerboseLevel(argv.verbose as string); + const timestamp = argv.timestamp as string; + const enableDebug = argv.enableDebug as boolean; const currentArgv: CLIOptions = { accounts, @@ -113,7 +116,9 @@ export class CLIService implements IService{ userComposeDir, blocklisting, startup, - verbose + verbose, + timestamp, + enableDebug }; return currentArgv; @@ -282,6 +287,15 @@ export class CLIService implements IService{ demandOption: false, choices: ['info', 'trace'], default: 'info', + }) + } + + private static enableDebugOption(yargs: Argv<{}>): void { + yargs.option('enable-debug', { + type: 'boolean', + describe: 'Enable or disable debugging of the local node', + demandOption: false, + default: false }); } diff --git a/src/services/record-parser/src/Parser.java b/src/services/record-parser/src/Parser.java index ae798a9e..5866a003 100644 --- a/src/services/record-parser/src/Parser.java +++ b/src/services/record-parser/src/Parser.java @@ -1,4 +1,4 @@ -import static com.hedera.services.utils.forensics.RecordParsers.parseV6RecordStreamEntriesIn; +import static com.hedera.node.app.service.mono.utils.forensics.RecordParsers.parseV6RecordStreamEntriesIn; public class Parser { public static void main(String[] args) { try { diff --git a/src/state/CleanUpState.ts b/src/state/CleanUpState.ts index 17cb8861..b51abf38 100644 --- a/src/state/CleanUpState.ts +++ b/src/state/CleanUpState.ts @@ -60,6 +60,7 @@ export class CleanUpState implements IState{ const application = yaml.load(readFileSync(propertiesFilePath).toString()) as any; delete application.hedera.mirror.importer.dataPath; delete application.hedera.mirror.importer.downloader.sources; + delete application.hedera.mirror.importer.downloader.local application.hedera.mirror.monitor.nodes = originalNodeConfiguration.fullNodeProperties; writeFileSync(propertiesFilePath, yaml.dump(application, { lineWidth: 256 })); diff --git a/src/state/DebugState.ts b/src/state/DebugState.ts index 06a1a213..50af4297 100644 --- a/src/state/DebugState.ts +++ b/src/state/DebugState.ts @@ -18,10 +18,16 @@ * */ +import { resolve } from 'path'; +import { readdirSync, copyFileSync, unlinkSync } from 'fs'; +import shell from 'shelljs'; import { IOBserver } from '../controller/IObserver'; import { LoggerService } from '../services/LoggerService'; import { ServiceLocator } from '../services/ServiceLocator'; import { IState } from './IState'; +import { CLIService } from '../services/CLIService'; +import { Errors } from '../Errors/LocalNodeErrors'; +import { RELATIVE_RECORDS_DIR_PATH, RELATIVE_TMP_DIR_PATH } from '../constants'; export class DebugState implements IState{ private logger: LoggerService; @@ -29,6 +35,9 @@ export class DebugState implements IState{ private observer: IOBserver | undefined; private stateName: string; + + private static readonly recordExt = `.${process.env.STREAM_EXTENSION}`; + private static readonly sigExt = `.${process.env.STREAM_SIG_EXTENSION}`; constructor() { this.stateName = DebugState.name; @@ -41,6 +50,104 @@ export class DebugState implements IState{ } public async onStart(): Promise { - throw new Error('Method not implemented.'); + try { + const { timestamp } = ServiceLocator.Current.get(CLIService.name).getCurrentArgv(); + // DebugState.checkForDebugMode(); + this.logger.trace('Debug State Starting...', this.stateName); + const jsTimestampNum = DebugState.getAndValidateTimestamp(timestamp) + + const tempDir = resolve(__dirname, RELATIVE_TMP_DIR_PATH); + const recordFilesDirPath = resolve(__dirname, RELATIVE_RECORDS_DIR_PATH); + this.findAndCopyRecordFileToTmpDir(jsTimestampNum, recordFilesDirPath, tempDir) + // Perform the parsing + await shell.exec( + 'docker exec network-node bash /opt/hgcapp/recordParser/parse.sh' + ); + + DebugState.cleanTempDir(tempDir); + } catch (error: any) { + this.logger.error(error.message); + return + } + } + + private static cleanTempDir(dirPath: string): void { + for (const tempFile of readdirSync(dirPath)) { + if (tempFile !== '.gitignore') { + unlinkSync(resolve(dirPath, tempFile)); + } + } + } + + private static getAndValidateTimestamp(timestamp: string): number { + const timestampRegEx = /^\d{10}[.-]\d{9}$/; + if (!timestampRegEx.test(timestamp)) { + throw Errors.INVALID_TIMESTAMP_ERROR(); + } + + // Parse the timestamp to a record file filename + let jsTimestamp = timestamp + .replace('.', '') + .replace('-', '') + .substring(0, 13); + return parseInt(jsTimestamp); + } + + private findAndCopyRecordFileToTmpDir(jsTimestampNum: number, recordFilesDirPath: string, tmpDirPath: string): void { + // Copy the record file to a temp directory + const files = readdirSync(recordFilesDirPath); + + for (let i = 1; i < files.length; i++) { + const file = files[i]; + const recordFileName = file.replace(DebugState.recordExt, ''); + const fileTimestamp = new Date(recordFileName.replace(/_/g, ':')).getTime(); + if (fileTimestamp >= jsTimestampNum) { + const fileToCopy = [ + files[i - 2], + files[i] + ]; + + this.copyFilesToTmpDir(fileToCopy, tmpDirPath, recordFilesDirPath) + return + } + } + + throw Errors.NO_RECORD_FILE_FOUND_ERROR(); + } + + private copyFilesToTmpDir( + filesToCopy: string | Array, + tmpDirPath: string, + recordFilesDirPath: string + ): void { + if (Array.isArray(filesToCopy)) { + for (const file of filesToCopy) { + this.copyFileToDir(file, recordFilesDirPath, tmpDirPath) + } + return + } + + this.copyFileToDir(filesToCopy, recordFilesDirPath, tmpDirPath) + } + + private copyFileToDir( + fileToCopy: string, + srcPath: string, + destinationPath: string, + ): void { + if (fileToCopy.endsWith(DebugState.recordExt)) { + this.logger.trace(`Parsing record file [${fileToCopy}]\n`); + } + + const fileToCopyName = fileToCopy.replace(DebugState.recordExt, ''); + const sigFile = fileToCopyName + DebugState.sigExt; + copyFileSync( + resolve(srcPath, fileToCopy), + resolve(destinationPath, fileToCopy) + ); + copyFileSync( + resolve(srcPath, sigFile), + resolve(destinationPath, sigFile) + ); } } diff --git a/src/state/InitState.ts b/src/state/InitState.ts index 82f62575..08dfed8a 100644 --- a/src/state/InitState.ts +++ b/src/state/InitState.ts @@ -130,6 +130,7 @@ export class InitState implements IState{ private configureMirrorNodeProperties() { this.logger.trace('Configuring required mirror node properties, depending on selected configuration...', this.stateName); const turboMode = !this.cliOptions.fullMode; + const debugMode = this.cliOptions.enableDebug; // const multiNode = this.cliOptions.multiNode; @@ -141,6 +142,10 @@ export class InitState implements IState{ application.hedera.mirror.importer.downloader.sources = originalNodeConfiguration.turboNodeProperties.sources; } + if (debugMode) { + application.hedera.mirror.importer.downloader.local = originalNodeConfiguration.local + } + // if (multiNode) { // application['hedera']['mirror']['monitor']['nodes'] = originalNodeConfiguration.multiNodeProperties // } diff --git a/src/state/StartState.ts b/src/state/StartState.ts index 5276daf7..732a550c 100644 --- a/src/state/StartState.ts +++ b/src/state/StartState.ts @@ -105,7 +105,7 @@ export class StartState implements IState{ } return shell.exec( - `docker compose -f ${composeFiles.join(' -f ')} up -d 2>${this.dockerService.getNullOutput()}` + `docker compose -f ${composeFiles.join(' -f ')} up -d 2>${this.dockerService.getNullOutput()}` ); } diff --git a/src/types/CLIOptions.ts b/src/types/CLIOptions.ts index b87f6454..6ff07fef 100644 --- a/src/types/CLIOptions.ts +++ b/src/types/CLIOptions.ts @@ -36,4 +36,6 @@ export interface CLIOptions { blocklisting: boolean, startup: boolean, verbose: number + timestamp: string, + enableDebug: boolean } diff --git a/src/utils/config.ts b/src/utils/config.ts new file mode 100644 index 00000000..8f98a769 --- /dev/null +++ b/src/utils/config.ts @@ -0,0 +1,33 @@ +/*- + * + * Hedera Local Node + * + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +import { join } from 'path'; +import yaml from 'js-yaml'; +import { readFileSync } from 'fs'; +import { APPLICATION_YML_RELATIVE_PATH } from '../constants'; + +export default function readApplicationYML() { + const propertiesFilePath = join(__dirname, APPLICATION_YML_RELATIVE_PATH); + const application = yaml.load(readFileSync(propertiesFilePath).toString()) as any; + + return { + propertiesFilePath, + application + } +}