Revert "cleanup: removed mock-node (#9654)" (#10069)

This reverts commit 7d7a18f, and then
removes the mock_node feature, which has been causing problems

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace.package]
-version = "0.0.0"                               # managed by cargo-workspaces, see below
+version = "0.0.0" # managed by cargo-workspaces, see below
 authors = ["Near Inc <[email protected]>"]
 edition = "2021"
 rust-version = "1.73.0"
@@ -69,6 +69,7 @@ members = [
     "tools/fork-network",
     "tools/indexer/example",
     "tools/mirror",
+    "tools/mock-node",
     "tools/ping",
     "tools/restaked",
     "tools/rpctypegen/core",
@@ -122,10 +123,7 @@ clap = { version = "4.2.0", features = ["derive", "env", "string"] }
 cloud-storage = "0.11.1"
 conqueue = "0.4.0"
 cpu-time = "1.0"
-criterion = { version = "0.3.5", default_features = false, features = [
-    "html_reports",
-    "cargo_bench_support",
-] }
+criterion = { version = "0.3.5", default_features = false, features = ["html_reports", "cargo_bench_support"] }
 crossbeam = "0.8"
 crossbeam-channel = "0.5.8"
 crossbeam-queue = "0.3.8"
@@ -175,7 +173,7 @@ more-asserts = "0.2"
 near-account-id = { version = "1.0.0-alpha.2", features = ["internal_unstable", "serde", "borsh"] }
 near-actix-test-utils = { path = "test-utils/actix-test-utils" }
 near-amend-genesis = { path = "tools/amend-genesis" }
-near-database-tool = { path = "tools/database" }
+near-database-tool= { path = "tools/database" }
 near-async = { path = "core/async" }
 near-cache = { path = "utils/near-cache" }
 near-chain = { path = "chain/chain" }
@@ -199,9 +197,7 @@ near-indexer-primitives = { path = "chain/indexer-primitives" }
 near-jsonrpc = { path = "chain/jsonrpc" }
 near-jsonrpc-adversarial-primitives = { path = "chain/jsonrpc-adversarial-primitives" }
 near-jsonrpc-client = { path = "chain/jsonrpc/client" }
-near-jsonrpc-primitives = { path = "chain/jsonrpc-primitives", features = [
-    "full",
-] }
+near-jsonrpc-primitives = { path = "chain/jsonrpc-primitives", features = ["full"] }
 near-jsonrpc-tests = { path = "chain/jsonrpc/jsonrpc-tests" }
 near-mainnet-res = { path = "utils/mainnet-res" }
 near-mirror = { path = "tools/mirror" }
@@ -276,13 +272,7 @@ reqwest = { version = "0.11.14", features = ["blocking"] }
 ripemd = "0.1.1"
 rkyv = "0.7.31"
 rlimit = "0.7"
-rocksdb = { version = "0.21.0", default-features = false, features = [
-    "snappy",
-    "lz4",
-    "zstd",
-    "zlib",
-    "jemalloc",
-] }
+rocksdb = { version = "0.21.0", default-features = false, features = ["snappy", "lz4", "zstd", "zlib", "jemalloc"] }
 runtime-tester = { path = "test-utils/runtime-tester" }
 rusqlite = { version = "0.29.0", features = ["bundled", "chrono", "functions"] }
 rustc-demangle = "0.1"
@@ -332,44 +322,27 @@ tracing = { version = "0.1.36", features = ["std"] }
 tracing-appender = "0.2.2"
 tracing-opentelemetry = "0.17.0"
 tracing-span-tree = "0.1"
-tracing-subscriber = { version = "0.3.15", features = [
-    "env-filter",
-    "fmt",
-    "registry",
-    "std",
-] }
+tracing-subscriber = { version = "0.3.15", features = ["env-filter", "fmt", "registry", "std"] }
 trybuild = "1.0.11"
 turn = "0.6"
 validator = "0.12"
 wasm-encoder = "0.27.0"
 wasmer-compiler = { package = "wasmer-compiler-near", version = "=2.4.1" }
 wasmer-compiler-singlepass = { package = "wasmer-compiler-singlepass-near", version = "=2.4.1" }
 wasmer-engine = { package = "wasmer-engine-near", version = "=2.4.1" }
-wasmer-engine-universal = { package = "wasmer-engine-universal-near", version = "=2.4.1", features = [
-    "compiler",
-] }
-wasmer-runtime = { version = "0.18.0", package = "wasmer-runtime-near", features = [
-    "default-backend-singlepass",
-], default-features = false }
+wasmer-engine-universal = { package = "wasmer-engine-universal-near", version = "=2.4.1", features = ["compiler"] }
+wasmer-runtime = { version = "0.18.0", package = "wasmer-runtime-near", features = ["default-backend-singlepass"], default-features = false }
 wasmer-runtime-core = { version = "0.18.2", package = "wasmer-runtime-core-near" }
 wasmer-types = { package = "wasmer-types-near", version = "=2.4.1" }
 wasmer-vm = { package = "wasmer-vm-near", version = "=2.4.1" }
 wasmparser = "0.78" # TODO: unify at least the versions of wasmparser we have in our codebase
 wasmprinter = "0.2"
 wasm-smith = "0.10"
-wasmtime = { version = "9.0.3", default-features = false, features = [
-    "cranelift",
-] }
+wasmtime = { version = "9.0.3", default-features = false, features = ["cranelift"] }
 wast = "40.0"
 wat = "1.0.40"
 webrtc-util = "0.7"
-winapi = { version = "0.3", features = [
-    "winbase",
-    "memoryapi",
-    "errhandlingapi",
-    "winnt",
-    "impl-default",
-] }
+winapi = { version = "0.3", features = ["winbase", "memoryapi", "errhandlingapi", "winnt", "impl-default"] }
 xshell = "0.2.1"
 xz2 = "0.1.6"
 

chain/chain/src/chain.rs

@@ -4052,6 +4052,7 @@ impl Chain {
 
         let protocol_version =
             self.epoch_manager.get_epoch_protocol_version(block.header().epoch_id())?;
+
         if checked_feature!("stable", AccessKeyNonceRange, protocol_version) {
             let transaction_validity_period = self.transaction_validity_period;
             for transaction in transactions {
@@ -4061,10 +4062,7 @@ impl Chain {
                         &transaction.transaction.block_hash,
                         transaction_validity_period,
                     )
-                    .map_err(|_| {
-                        tracing::warn!("Invalid Transactions for mock node");
-                        Error::from(Error::InvalidTransactions)
-                    })?;
+                    .map_err(|_| Error::from(Error::InvalidTransactions))?;
             }
         };
 

tools/mock-node/Cargo.toml

@@ -0,0 +1,48 @@
+[package]
+name = "mock-node"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+license.workspace = true
+publish = false
+
+[dependencies]
+actix-rt.workspace = true
+actix.workspace = true
+anyhow.workspace = true
+clap.workspace = true
+futures.workspace = true
+pin-project.workspace = true
+rand.workspace = true
+rayon.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+tempfile.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+
+near-actix-test-utils.workspace = true
+near-async.workspace = true
+near-chain.workspace = true
+near-chain-configs.workspace = true
+near-client.workspace = true
+near-crypto.workspace = true
+near-chunks.workspace = true
+near-epoch-manager.workspace = true
+near-jsonrpc.workspace = true
+near-jsonrpc-client.workspace = true
+near-network.workspace = true
+near-store.workspace = true
+near-o11y.workspace = true
+near-telemetry.workspace = true
+near-performance-metrics.workspace = true
+near-primitives.workspace = true
+nearcore.workspace = true
+
+[[bin]]
+name = "mock-node"
+
+[features]
+test_features = ["nearcore/test_features"]

tools/mock-node/README.md

@@ -0,0 +1,90 @@
+# mock-node
+This crate hosts libraries to start a test env for a single node by replacing the network module with a mock network environment.
+The mock network environment simulates the interaction that the client will usually have with other nodes by
+responding to the client's network messages and broadcasting new blocks. The mock network reads a pre-generated chain
+history from storage.
+
+## Quick Start
+
+```console
+$ cargo run --release -p mock-node -- ~/.near/localnet/node0
+```
+
+where the `node0` directory contains some pre-generated chain history in storage.
+You can find two examples in the ./benches directory.
+
+If you are running a mock node for mainnet or testnet on a GCP node, you want to place the new client home
+dir on a SSD disk for optimal rocksdb performance. Note that the
+default booting disk of GCP notes are HDD, so you need to mount a new SSD disk on
+your node and put the mock node's home dir there. See https://cloud.google.com/compute/docs/disks/add-persistent-disk
+for how to attach and mount a new disk to an existing GCP node.
+
+See `$ cargo run -p mock-node -- --help` for the list of available options and their documentation.
+
+## Examples
+
+#### Replay localnet history
+
+```console
+$ cargo r -r -p mock-node -- ~/.near/localnet/node0
+```
+Here we take the home dir of an existing node in a localnet as chain history home dir,
+so the mock network will reproduce the client catching up with the entire history of the localnet from genesis.
+
+#### Replay mainnet history from a certain height
+
+To replay mainnet or testnet history, in most use cases, we want to start replaying from a certain height, instead
+of from genesis block. The following comment replays mainnet history from block height 60925880 to block height 60925900.
+
+```console
+$ cargo r -r -p mock-node --  ~/.near ~/mock_node_home_dir --start_height 60925880 --target-height 60925900
+```
+
+By providing a starting height,
+the binary will set up the data dir before starting the client, by copying the state at the specified height
+and other chain info necessary for processing the blocks afterwards (such as block headers and blocks).
+This initial setup may take a long time (The exact depends on your
+source dir, my experiment takes about an hour from a non-archival source dir. Copying from archival node source
+dir may take longer as rocksdb is slower). So we suggest specifying a client dir (the `~/mock_node_home_dir` argument)
+so you can reuse it again without having to copy the state again.
+
+Note that the start height must be the last block of an epoch.
+
+Once you have the source dir already set up, you can run the command without `--start_height`,
+
+```console
+$  cargo r -r -p mock-node --  ~/.near ~/mock_node_home_dir --target-height 60926000
+```
+Without `--starting_height`, the binary will not modify the client home dir before starting the mock node. Therefore,
+the mock node will start from the chain head stored in the client dir.
+
+## Mock Network Configuration
+
+Certain details around how the mock network behaves can be configured with the file `mock.json` in the chain history
+home directory. Currently, the only supported configuration options tell how long to wait before replying to requests
+(the same as the --network_delay flag), and how often to send unrequested blocks and chunk part requests. By default,
+no such unrequested messages are sent, but the following config file will have the mock code produce unrequested
+blocks every 100 milliseconds, and chunk part requests every 50 milliseconds.
+
+```json
+{
+    "response_delay": {
+        "secs": 0,
+        "nanos": 100000000
+    },
+    "incoming_requests": {
+        "block": {
+            "interval": {
+                "secs": 0,
+                "nanos": 100000000
+            }
+        },
+        "chunk_request": {
+            "interval": {
+                "secs": 0,
+                "nanos": 50000000
+            }
+        }
+    }
+}
+```