From c4ebb5920f7ac931b944506b9c003ca99a66f863 Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Tue, 3 Dec 2024 15:41:43 +0200 Subject: [PATCH] apply-range: add ability to operate with memtries & a new benchmark mode (#12522) With this PR the original `benchmarking` functionality has been subsumed by the `sequential` mode which has been extended to operate on flat and memtrie storage modes. `benchmark` is then changed to not commit any results to the storage, effectively leading to the same starting block being applied over and over again, thus ideally demonstrating the time spent inside `Runtime::apply`. I've seen this to sometimes handle a huge number of blocks per second which would make the output somewhat difficult keep track of, so I went ahead and integrated indicatif-based progress bar (which then also comes with a built-in mechanism to estimate duration and rate of blocks per second.) --- Cargo.lock | 96 +++------ Cargo.toml | 2 +- deny.toml | 3 + genesis-tools/genesis-populate/src/lib.rs | 2 +- nearcore/src/download_file.rs | 5 +- tools/flat-storage/src/commands.rs | 7 +- tools/replay-archive/Cargo.toml | 1 + tools/replay-archive/src/cli.rs | 10 +- tools/state-viewer/Cargo.toml | 1 + tools/state-viewer/src/apply_chain_range.rs | 214 ++++++++++++-------- tools/state-viewer/src/cli.rs | 18 +- tools/state-viewer/src/progress_reporter.rs | 49 +++-- 12 files changed, 219 insertions(+), 189 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8f22530ff92..36ed8c8a503 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1423,7 +1423,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" dependencies = [ "termcolor", - "unicode-width", + "unicode-width 0.1.9", ] [[package]] @@ -1473,15 +1473,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.5" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", - "unicode-width", - "windows-sys 0.42.0", + "unicode-width 0.1.9", + "windows-sys 0.52.0", ] [[package]] @@ -3171,15 +3171,16 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.15.0" +version = "0.17.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" dependencies = [ "console", - "lazy_static", "number_prefix", + "portable-atomic", "rayon", - "regex", + "unicode-width 0.2.0", + "web-time", ] [[package]] @@ -4747,6 +4748,7 @@ dependencies = [ "anyhow", "borsh", "clap", + "indicatif", "itertools 0.12.1", "near-chain", "near-chain-configs", @@ -5545,9 +5547,9 @@ dependencies = [ [[package]] name = "number_prefix" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" @@ -6048,6 +6050,12 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + [[package]] name = "postcard" version = "1.0.10" @@ -7466,6 +7474,7 @@ dependencies = [ "chrono", "clap", "cloud-storage", + "indicatif", "insta", "itertools 0.12.1", "near-chain", @@ -8282,6 +8291,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -8896,7 +8911,7 @@ checksum = "9bb4f48a8b083dbc50e291e430afb8f524092bb00428957bcc63f49f856c64ac" dependencies = [ "leb128", "memchr", - "unicode-width", + "unicode-width 0.1.9", ] [[package]] @@ -9019,21 +9034,6 @@ dependencies = [ "windows_x86_64_msvc 0.36.1", ] -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -9092,12 +9092,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.0" @@ -9116,12 +9110,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.0" @@ -9140,12 +9128,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.0" @@ -9170,12 +9152,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.0" @@ -9194,12 +9170,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.0" @@ -9212,12 +9182,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" @@ -9236,12 +9200,6 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index dc1acbc999b..98ef7b882ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -199,7 +199,7 @@ hyper = { version = "0.14", features = ["full"] } hyper-tls = "0.5.0" im = "15" indexmap = "2" -indicatif = { version = "0.15.0", features = ["with_rayon"] } +indicatif = { version = "0.17.0", features = ["rayon"] } insta = { version = "1.41.0", features = ["json", "yaml", "redactions"] } integration-tests = { path = "integration-tests" } inventory = "0.3.15" diff --git a/deny.toml b/deny.toml index d772d7acbf2..443e010e348 100644 --- a/deny.toml +++ b/deny.toml @@ -136,4 +136,7 @@ skip = [ { name = "thiserror", version = "<2.0" }, { name = "thiserror-impl", version = "<2.0" }, { name = "derive_more", version = "<1" }, + + # indicatif brings in a newer version + { name = "unicode-width", version = "<0.2" }, ] diff --git a/genesis-tools/genesis-populate/src/lib.rs b/genesis-tools/genesis-populate/src/lib.rs index aba0ec31292..fa8d40c70fe 100644 --- a/genesis-tools/genesis-populate/src/lib.rs +++ b/genesis-tools/genesis-populate/src/lib.rs @@ -164,7 +164,7 @@ impl GenesisBuilder { let bar = ProgressBar::new(total_accounts_num as _); bar.set_style(ProgressStyle::default_bar().template( "[elapsed {elapsed_precise} remaining {eta_precise}] Writing into storage {bar} {pos:>7}/{len:7}", - )); + ).unwrap()); // Add records in chunks of 3000 per shard for memory efficiency reasons. for i in 0..total_accounts_num { let account_id = get_account_id(i); diff --git a/nearcore/src/download_file.rs b/nearcore/src/download_file.rs index ae2dc794066..025d039b529 100644 --- a/nearcore/src/download_file.rs +++ b/nearcore/src/download_file.rs @@ -56,14 +56,15 @@ async fn download_file_impl( bar.set_style( ProgressStyle::default_bar().template( "{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} [{bytes_per_sec}] ({eta})" - ).progress_chars("#>-") + ).unwrap().progress_chars("#>-") ); bar } else { let bar = ProgressBar::new_spinner(); bar.set_style( ProgressStyle::default_bar() - .template("{spinner:.green} [{elapsed_precise}] {bytes} [{bytes_per_sec}]"), + .template("{spinner:.green} [{elapsed_precise}] {bytes} [{bytes_per_sec}]") + .unwrap(), ); bar }; diff --git a/tools/flat-storage/src/commands.rs b/tools/flat-storage/src/commands.rs index 9a55700a7a9..314e8e406e4 100644 --- a/tools/flat-storage/src/commands.rs +++ b/tools/flat-storage/src/commands.rs @@ -46,7 +46,7 @@ enum SubCommand { /// The trie is constructed for the block height equal to flat_head ConstructTrieFromFlat(ConstructTriedFromFlatCmd), - /// Move flat head forward. + /// Move flat head forward or backward. MoveFlatHead(MoveFlatHeadCmd), } @@ -106,8 +106,9 @@ pub enum MoveFlatHeadMode { new_flat_head_height: BlockHeight, }, /// Moves head back by specific number of blocks. - /// Note: it doesn't record deltas on the way and should be used - /// only for replaying chain forward. + /// + /// Note: it doesn't record deltas on the way and should be used only for replaying chain + /// forward. Back { #[clap(long)] blocks: usize, diff --git a/tools/replay-archive/Cargo.toml b/tools/replay-archive/Cargo.toml index af5c30dd5ae..7e0279015bc 100644 --- a/tools/replay-archive/Cargo.toml +++ b/tools/replay-archive/Cargo.toml @@ -16,6 +16,7 @@ anyhow.workspace = true borsh.workspace = true clap.workspace = true itertools.workspace = true +indicatif.workspace = true near-chain.workspace = true near-chain-primitives.workspace = true diff --git a/tools/replay-archive/src/cli.rs b/tools/replay-archive/src/cli.rs index a04a13b0c23..b6ad0216494 100644 --- a/tools/replay-archive/src/cli.rs +++ b/tools/replay-archive/src/cli.rs @@ -27,7 +27,7 @@ use near_primitives::sharding::{ReceiptProof, ShardChunk, ShardChunkHeader, Shar use near_primitives::types::chunk_extra::ChunkExtra; use near_primitives::types::{BlockHeight, Gas, ProtocolVersion, ShardId}; use near_primitives::version::ProtocolFeature; -use near_state_viewer::progress_reporter::{timestamp_ms, ProgressReporter}; +use near_state_viewer::progress_reporter::ProgressReporter; use near_store::{get_genesis_state_roots, ShardUId, Store}; use nearcore::{load_config, NearConfig, NightshadeRuntime, NightshadeRuntimeExt}; use std::collections::HashMap; @@ -132,12 +132,13 @@ impl ReplayController { let progress_reporter = ProgressReporter { cnt: AtomicU64::new(0), - ts: AtomicU64::new(timestamp_ms()), - all: (end_height + 1).saturating_sub(start_height), skipped: AtomicU64::new(0), empty_blocks: AtomicU64::new(0), non_empty_blocks: AtomicU64::new(0), tgas_burned: AtomicU64::new(0), + indicatif: near_state_viewer::progress_reporter::default_indicatif( + (end_height + 1).checked_sub(start_height), + ), }; Ok(Self { @@ -192,7 +193,8 @@ impl ReplayController { total_gas_burnt = Some(gas_burnt); } } - self.progress_reporter.inc_and_report_progress(total_gas_burnt.unwrap_or(0)); + self.progress_reporter + .inc_and_report_progress(self.next_height, total_gas_burnt.unwrap_or(0)); self.next_height += 1; Ok(self.next_height <= self.end_height) } diff --git a/tools/state-viewer/Cargo.toml b/tools/state-viewer/Cargo.toml index 7dfc8cb10e4..58b76941ed3 100644 --- a/tools/state-viewer/Cargo.toml +++ b/tools/state-viewer/Cargo.toml @@ -33,6 +33,7 @@ tempfile.workspace = true thiserror.workspace = true tracing.workspace = true yansi.workspace = true +indicatif.workspace = true near-time.workspace = true near-chain-configs.workspace = true diff --git a/tools/state-viewer/src/apply_chain_range.rs b/tools/state-viewer/src/apply_chain_range.rs index e8f46c7b620..bb4fbedd7c5 100644 --- a/tools/state-viewer/src/apply_chain_range.rs +++ b/tools/state-viewer/src/apply_chain_range.rs @@ -1,6 +1,6 @@ use crate::cli::{ApplyRangeMode, StorageSource}; use crate::commands::{maybe_print_db_stats, maybe_save_trie_changes}; -use crate::progress_reporter::{timestamp_ms, ProgressReporter}; +use crate::progress_reporter::ProgressReporter; use near_chain::chain::collect_receipts_from_response; use near_chain::migrations::check_if_block_is_first_with_chunk_of_version; use near_chain::types::{ @@ -78,7 +78,7 @@ fn apply_block_from_range( Ok(block_hash) => block_hash, Err(_) => { // Skipping block because it's not available in ChainStore. - progress_reporter.inc_and_report_progress(0); + progress_reporter.inc_and_report_progress(height, 0); return; } }; @@ -101,7 +101,7 @@ fn apply_block_from_range( if verbose_output { println!("Skipping the genesis block #{}.", height); } - progress_reporter.inc_and_report_progress(0); + progress_reporter.inc_and_report_progress(height, 0); return; } else if block.chunks()[shard_index].height_included() == height { chunk_present = true; @@ -137,7 +137,7 @@ fn apply_block_from_range( chunk_present ), ); - progress_reporter.inc_and_report_progress(0); + progress_reporter.inc_and_report_progress(height, 0); return; } }; @@ -286,43 +286,60 @@ fn apply_block_from_range( apply_result.trie_changes.state_changes().len(), ), ); - progress_reporter.inc_and_report_progress(apply_result.total_gas_burnt); - - if mode == ApplyRangeMode::Benchmarking { - // Compute delta and immediately apply to flat storage. - let changes = - FlatStateChanges::from_state_changes(apply_result.trie_changes.state_changes()); - let delta = near_store::flat::FlatStateDelta { - metadata: near_store::flat::FlatStateDeltaMetadata { - block: BlockInfo { - hash: block_hash, - height: block.header().height(), - prev_hash: *block.header().prev_hash(), + progress_reporter.inc_and_report_progress(height, apply_result.total_gas_burnt); + + // See documentation for `ApplyRangeMode` variants. + // + // Ultimately, this has to handle requirements on storage effects from multiple sources -- + // `Benchmark` for example repeatedly applies a single block, so no storage effects are + // desired, meanwhile other modes can be set to operate on various storage sources, all of + // which have their unique propoerties (e.g. flat storage operates on flat_head...) + match (mode, storage) { + (ApplyRangeMode::Benchmark, _) => {} + (_, StorageSource::Trie | StorageSource::TrieFree) => {} + (_, StorageSource::FlatStorage | StorageSource::Memtrie) => { + // Compute delta and immediately apply to flat storage. + let changes = + FlatStateChanges::from_state_changes(apply_result.trie_changes.state_changes()); + let delta = near_store::flat::FlatStateDelta { + metadata: near_store::flat::FlatStateDeltaMetadata { + block: BlockInfo { + hash: block_hash, + height: block.header().height(), + prev_hash: *block.header().prev_hash(), + }, + prev_block_with_changes: None, }, - prev_block_with_changes: None, - }, - changes, - }; - - let flat_storage_manager = runtime_adapter.get_flat_storage_manager(); - let flat_storage = flat_storage_manager.get_flat_storage_for_shard(shard_uid).unwrap(); - let store_update = flat_storage.add_delta(delta).unwrap(); - store_update.commit().unwrap(); - flat_storage.update_flat_head(&block_hash).unwrap(); + changes, + }; - // Apply trie changes to trie node caches. - let mut fake_store_update = read_store.trie_store().store_update(); - apply_result.trie_changes.insertions_into(&mut fake_store_update); - apply_result.trie_changes.deletions_into(&mut fake_store_update); - } else { - if let Err(err) = maybe_save_trie_changes( - write_store, - genesis.config.genesis_height, - apply_result, - height, - shard_id, - ) { - panic!("Error while saving trie changes at height {height}, shard {shard_id} ({err})"); + let flat_storage_manager = runtime_adapter.get_flat_storage_manager(); + let flat_storage = flat_storage_manager.get_flat_storage_for_shard(shard_uid).unwrap(); + let store_update = flat_storage.add_delta(delta).unwrap(); + store_update.commit().unwrap(); + flat_storage.update_flat_head(&block_hash).unwrap(); + } + } + match (mode, storage) { + (ApplyRangeMode::Benchmark, _) => {} + (_, StorageSource::FlatStorage) => { + // Apply trie changes to trie node caches. + let mut fake_store_update = read_store.trie_store().store_update(); + apply_result.trie_changes.insertions_into(&mut fake_store_update); + apply_result.trie_changes.deletions_into(&mut fake_store_update); + } + (_, StorageSource::Trie | StorageSource::TrieFree | StorageSource::Memtrie) => { + if let Err(err) = maybe_save_trie_changes( + write_store, + genesis.config.genesis_height, + apply_result, + height, + shard_id, + ) { + panic!( + "Error while saving trie changes at height {height}, shard {shard_id} ({err})" + ); + } } } } @@ -353,59 +370,81 @@ pub fn apply_chain_range( ?storage) .entered(); let chain_store = ChainStore::new(read_store.clone(), genesis.config.genesis_height, false); - let (start_height, end_height) = match mode { - ApplyRangeMode::Benchmarking => { - // Benchmarking mode requires flat storage and retrieves start and - // end heights from flat storage and chain. - assert!(matches!(storage, StorageSource::FlatStorage)); - assert!(start_height.is_none()); - assert!(end_height.is_none()); - - let chain_store = - ChainStore::new(read_store.clone(), genesis.config.genesis_height, false); - let final_head = chain_store.final_head().unwrap(); - let shard_layout = epoch_manager.get_shard_layout(&final_head.epoch_id).unwrap(); - let shard_uid = near_primitives::shard_layout::ShardUId::from_shard_id_and_layout( - shard_id, - &shard_layout, - ); - let flat_head = match read_store.flat_store().get_flat_storage_status(shard_uid) { - Ok(FlatStorageStatus::Ready(ready_status)) => ready_status.flat_head, - status => { - panic!("cannot create flat storage for shard {shard_id} with status {status:?}") - } - }; + let final_head = chain_store.final_head().unwrap(); + let shard_layout = epoch_manager.get_shard_layout(&final_head.epoch_id).unwrap(); + let shard_uid = + near_primitives::shard_layout::ShardUId::from_shard_id_and_layout(shard_id, &shard_layout); + + // Load the requested type of storage for transactions and actions to act upon. This may allow + // configurations that aren't used in production anymore, in case anybody really wants that + // behaviour. + match storage { + StorageSource::Trie | StorageSource::TrieFree => {} + StorageSource::FlatStorage => { let flat_storage_manager = runtime_adapter.get_flat_storage_manager(); flat_storage_manager.create_flat_storage_for_shard(shard_uid).unwrap(); + } + StorageSource::Memtrie => { + // Memtries require flat storage to load. + let flat_storage_manager = runtime_adapter.get_flat_storage_manager(); + flat_storage_manager.create_flat_storage_for_shard(shard_uid).unwrap(); + runtime_adapter + .get_tries() + .load_mem_trie(&shard_uid, None, true) + .expect("load mem trie"); + } + } - // Note that first height to apply is the first one after flat - // head. - (flat_head.height + 1, final_head.height) + let (start_height, end_height) = match (mode, storage) { + (ApplyRangeMode::Benchmark, StorageSource::Trie | StorageSource::TrieFree) => { + panic!("benchmark with --storage trie|trie-free is not supported") } - _ => ( + (ApplyRangeMode::Benchmark, StorageSource::FlatStorage | StorageSource::Memtrie) => { + // Benchmarking mode requires flat storage and retrieves the block height from flat + // storage. + assert!(start_height.is_none()); + assert!(end_height.is_none()); + let flat_status = read_store.flat_store().get_flat_storage_status(shard_uid); + let Ok(FlatStorageStatus::Ready(ready)) = flat_status else { + panic!("cannot create flat storage for shard {shard_uid} due to {flat_status:?}") + }; + // We apply the block at flat_head. Users can set the block they want to benchmark by + // moving the flat head using the `flat-storage move-flat-head` command. End point of + // `0` helps indicatif to display more reasonable output. + (ready.flat_head.height + 1, 0) + } + (_, StorageSource::Trie | StorageSource::TrieFree) => ( start_height.unwrap_or_else(|| chain_store.tail().unwrap()), end_height.unwrap_or_else(|| chain_store.head().unwrap().height), ), + (_, StorageSource::FlatStorage | StorageSource::Memtrie) => { + let start_height = start_height.unwrap_or_else(|| { + let status = read_store.flat_store().get_flat_storage_status(shard_uid); + let Ok(FlatStorageStatus::Ready(ready)) = status else { + panic!("cannot create flat storage for shard {shard_uid} due to {status:?}") + }; + ready.flat_head.height + 1 + }); + (start_height, end_height.unwrap_or_else(|| chain_store.head().unwrap().height)) + } }; - println!( - "Applying chunks in the range {}..={} for shard_id {}", - start_height, end_height, shard_id - ); - - println!("Printing results including outcomes of applying receipts"); + let range = start_height..=end_height; + println!("Applying chunks in the range {range:?} for {shard_uid}…"); + if csv_file.is_some() { + println!("Writing results of applying receipts to the CSV file"); + } let csv_file_mutex = Mutex::new(csv_file); maybe_add_to_csv(&csv_file_mutex, "Height,Hash,Author,#Tx,#Receipt,Timestamp,GasUsed,ChunkPresent,#ProcessedDelayedReceipts,#DelayedReceipts,#StateChanges"); - - let range = start_height..=end_height; let progress_reporter = ProgressReporter { cnt: AtomicU64::new(0), - ts: AtomicU64::new(timestamp_ms()), - all: (end_height + 1).saturating_sub(start_height), skipped: AtomicU64::new(0), empty_blocks: AtomicU64::new(0), non_empty_blocks: AtomicU64::new(0), tgas_burned: AtomicU64::new(0), + indicatif: crate::progress_reporter::default_indicatif( + (end_height + 1).checked_sub(start_height), + ), }; let process_height = |height| { apply_block_from_range( @@ -425,34 +464,45 @@ pub fn apply_chain_range( ); }; + let start_time = near_time::Instant::now(); match mode { - ApplyRangeMode::Sequential | ApplyRangeMode::Benchmarking => { - range.into_iter().for_each(|height| { + ApplyRangeMode::Sequential => { + range.clone().into_iter().for_each(|height| { let _span = tracing::debug_span!( target: "state_viewer", parent: &parent_span, - "process_block_in_order", + "process_block", height) .entered(); process_height(height) }); } ApplyRangeMode::Parallel => { - range.into_par_iter().for_each(|height| { + range.clone().into_par_iter().for_each(|height| { let _span = tracing::debug_span!( target: "mock_node", parent: &parent_span, - "process_block_in_parallel", + "process_block", height) .entered(); process_height(height) }); } + ApplyRangeMode::Benchmark => loop { + let height = range.start(); + let _span = tracing::debug_span!( + target: "state_viewer", + parent: &parent_span, + "process_block", + height) + .entered(); + process_height(*height) + }, } println!( - "No differences found after applying chunks in the range {}..={} for shard_id {}", - start_height, end_height, shard_id + "Applied range {range:?} for shard {shard_uid} in {elapsed:?}", + elapsed = start_time.elapsed() ); } diff --git a/tools/state-viewer/src/cli.rs b/tools/state-viewer/src/cli.rs index 74181882035..90a11b875b1 100644 --- a/tools/state-viewer/src/cli.rs +++ b/tools/state-viewer/src/cli.rs @@ -208,7 +208,10 @@ pub enum StorageSource { /// Use the data stored in trie, but without paying extra gas costs. /// This could be used to simulate flat storage when the latter is not present. TrieFree, + #[value(alias("flat"))] FlatStorage, + /// Implies flat storage and loads the memtries as well. + Memtrie, } impl StorageSource { @@ -217,6 +220,9 @@ impl StorageSource { StorageSource::Trie => RuntimeStorageConfig::new(state_root, false), StorageSource::TrieFree => RuntimeStorageConfig::new_with_db_trie_only(state_root), StorageSource::FlatStorage => RuntimeStorageConfig::new(state_root, true), + // This is the same as FlatStorage handling. That's because memtrie initialization + // happens as part of `ShardTries::load_mem_trie` function call. + StorageSource::Memtrie => RuntimeStorageConfig::new(state_root, true), } } } @@ -283,16 +289,16 @@ impl ApplyChunkCmd { #[derive(clap::Parser, Copy, Clone, Debug, Eq, PartialEq)] pub enum ApplyRangeMode { /// Applies chunks one after another in order of increasing heights. + /// + /// Great for profiling. Sequential, /// Applies chunks in parallel. + /// /// Useful for quick correctness check of applying chunks by comparing /// results with `ChunkExtra`s. Parallel, - /// Sequentially applies chunks from flat storage head until chain - /// final head, moving flat head forward. Use in combination with - /// `MoveFlatHeadCmd` and `MoveFlatHeadMode::Back`. - /// Useful for benchmarking. - Benchmarking, + /// Applies a single block repeatedly without committing any state changes. + Benchmark, } #[derive(clap::Parser)] @@ -326,7 +332,7 @@ impl ApplyRangeCmd { store: Store, node_storage: NodeStorage, ) { - if matches!(self.mode, ApplyRangeMode::Benchmarking) && self.save_state.is_some() { + if matches!(self.mode, ApplyRangeMode::Benchmark) && self.save_state.is_some() { panic!("Persisting trie nodes in storage is not compatible with benchmark mode!"); } apply_range( diff --git a/tools/state-viewer/src/progress_reporter.rs b/tools/state-viewer/src/progress_reporter.rs index 781f4dcd8a1..e7ea56f2fcb 100644 --- a/tools/state-viewer/src/progress_reporter.rs +++ b/tools/state-viewer/src/progress_reporter.rs @@ -7,23 +7,36 @@ pub fn timestamp_ms() -> u64 { const TGAS: u64 = 1024 * 1024 * 1024 * 1024; +pub fn default_indicatif(len: Option) -> indicatif::ProgressBar { + indicatif::ProgressBar::with_draw_target( + len, + indicatif::ProgressDrawTarget::stderr_with_hz(5) + ).with_style(indicatif::ProgressStyle::with_template( + "{prefix}{pos}/{len} blocks applied in {elapsed} at a rate of {per_sec}. {eta} remaining. {msg}" + ).unwrap()) +} + pub struct ProgressReporter { pub cnt: AtomicU64, - // Timestamp to make relative measurements of block processing speed (in ms) - pub ts: AtomicU64, - pub all: u64, pub skipped: AtomicU64, // Fields below get cleared after each print. pub empty_blocks: AtomicU64, pub non_empty_blocks: AtomicU64, // Total gas burned (in TGas) pub tgas_burned: AtomicU64, + pub indicatif: indicatif::ProgressBar, } impl ProgressReporter { - pub fn inc_and_report_progress(&self, gas_burnt: u64) { - let ProgressReporter { cnt, ts, all, skipped, empty_blocks, non_empty_blocks, tgas_burned } = - self; + pub fn inc_and_report_progress(&self, block_height: u64, gas_burnt: u64) { + let ProgressReporter { + cnt, + skipped, + empty_blocks, + non_empty_blocks, + tgas_burned, + indicatif, + } = self; if gas_burnt == 0 { empty_blocks.fetch_add(1, Ordering::Relaxed); } else { @@ -33,12 +46,9 @@ impl ProgressReporter { const PRINT_PER: u64 = 100; let prev = cnt.fetch_add(1, Ordering::Relaxed); - if (prev + 1) % PRINT_PER == 0 { - let prev_ts = ts.load(Ordering::Relaxed); - let new_ts = timestamp_ms(); - let per_second = (PRINT_PER as f64 / (new_ts - prev_ts) as f64) * 1000.0; - ts.store(new_ts, Ordering::Relaxed); - let secs_remaining = (all - prev) as f64 / per_second; + let current = 1 + prev; + indicatif.set_position(current); + if current % PRINT_PER == 0 { let avg_gas = if non_empty_blocks.load(Ordering::Relaxed) == 0 { 0.0 } else { @@ -46,15 +56,12 @@ impl ProgressReporter { / non_empty_blocks.load(Ordering::Relaxed) as f64 }; - println!( - "Processed {} blocks, {:.4} blocks per second ({} skipped), {:.2} secs remaining {} empty blocks {:.2} avg gas per non-empty block", - prev + 1, - per_second, - skipped.load(Ordering::Relaxed), - secs_remaining, - empty_blocks.load(Ordering::Relaxed), - avg_gas, - ); + indicatif.set_message(format!( + "Skipped {skipped} blocks. \ + Over last 100 blocks to height {block_height}: {empty} empty blocks, averaging {avg_gas:.2} Tgas per non-empty block", + skipped = skipped.load(Ordering::Relaxed), + empty = empty_blocks.load(Ordering::Relaxed), + )); empty_blocks.store(0, Ordering::Relaxed); non_empty_blocks.store(0, Ordering::Relaxed); tgas_burned.store(0, Ordering::Relaxed);