From 339f473a256c05d36ac644c930a080cca2b65b86 Mon Sep 17 00:00:00 2001 From: Vladimir Petrzhikovskii Date: Wed, 23 Oct 2024 19:22:00 +0200 Subject: [PATCH] fix: account for cells tree size in block cache weigher --- block-util/src/block/block_stuff.rs | 28 ++++++++++++++++++++++++---- collator/src/collator/build_block.rs | 16 ++++++++++++++-- storage/src/store/block/mod.rs | 16 +++++++++++++--- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/block-util/src/block/block_stuff.rs b/block-util/src/block/block_stuff.rs index 640c92742..641d7a88d 100644 --- a/block-util/src/block/block_stuff.rs +++ b/block-util/src/block/block_stuff.rs @@ -1,8 +1,9 @@ use std::sync::{Arc, OnceLock}; use std::time::Duration; -use anyhow::Result; +use anyhow::{Context, Result}; use bytes::Bytes; +use everscale_types::boc::de::BocHeader; use everscale_types::models::*; use everscale_types::prelude::*; use tycho_util::FastHashMap; @@ -70,10 +71,16 @@ impl BlockStuff { file_hash, }; - Self::from_block_and_root(&block_id, block, root, DATA_SIZE) + Self::from_block_and_root(&block_id, block, root, DATA_SIZE, 0) } - pub fn from_block_and_root(id: &BlockId, block: Block, root: Cell, data_size: usize) -> Self { + pub fn from_block_and_root( + id: &BlockId, + block: Block, + root: Cell, + data_size: usize, + uniq_cells: usize, + ) -> Self { debug_assert_eq!(&id.root_hash, root.repr_hash()); Self { @@ -85,6 +92,7 @@ impl BlockStuff { block_extra: Default::default(), block_mc_extra: Default::default(), data_size, + uniq_cells, }), } } @@ -100,7 +108,13 @@ impl BlockStuff { } pub fn deserialize(id: &BlockId, data: &[u8]) -> Result { - let root = Boc::decode(data)?; + let root = BocHeader::decode(data, &Default::default())?; + let uniq_cells = root.cells().len(); + let root = root.finalize(&mut Cell::empty_context())?; + let root = root + .get(0) + .context("Invalid blocK: failed to get root cell")?; + anyhow::ensure!( &id.root_hash == root.repr_hash(), "root_hash mismatch for {id}" @@ -116,6 +130,7 @@ impl BlockStuff { block_extra: Default::default(), block_mc_extra: Default::default(), data_size: data.len(), + uniq_cells, }), }) } @@ -128,6 +143,10 @@ impl BlockStuff { self.inner.data_size } + pub fn num_uniq_cells(&self) -> usize { + self.inner.uniq_cells + } + pub fn with_archive_data(self, data: A) -> WithArchiveData where Bytes: From, @@ -286,4 +305,5 @@ pub struct Inner { block_extra: OnceLock>, block_mc_extra: OnceLock>, data_size: usize, + uniq_cells: usize, } diff --git a/collator/src/collator/build_block.rs b/collator/src/collator/build_block.rs index 22cdf4620..2831ca10b 100644 --- a/collator/src/collator/build_block.rs +++ b/collator/src/collator/build_block.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, bail, Result}; +use everscale_types::boc::ser::BocHeader; use everscale_types::merkle::*; use everscale_types::models::*; use everscale_types::prelude::*; @@ -327,7 +328,12 @@ impl CollatorStdImpl { // TODO: Check (assert) whether the serialized block contains usage cells let root = CellBuilder::build_from(&block)?; - let data = everscale_types::boc::Boc::encode_rayon(&root); + let mut data = Vec::with_capacity(50_000_000); // todo: config based on some metrics + let header = BocHeader::::with_root(root.as_ref()); + header.encode_rayon(&mut data); + + let num_uniq_cells = header.compute_stats().total_cells_size; // todo: expose root.len() in everscale_types + let block_id = BlockId { shard: collation_data.block_id_short.shard, seqno: collation_data.block_id_short.seqno, @@ -337,7 +343,13 @@ impl CollatorStdImpl { build_block_elapsed = histogram.finish(); - let block = BlockStuff::from_block_and_root(&block_id, block, root, data.len()); + let block = BlockStuff::from_block_and_root( + &block_id, + block, + root, + data.len(), + num_uniq_cells as usize, + ); ( WithArchiveData::new(block, data), diff --git a/storage/src/store/block/mod.rs b/storage/src/store/block/mod.rs index db83758d4..060120d2d 100644 --- a/storage/src/store/block/mod.rs +++ b/storage/src/store/block/mod.rs @@ -60,9 +60,19 @@ impl BlockStorage { fn weigher(_key: &BlockId, value: &BlockStuff) -> u32 { const BLOCK_STUFF_OVERHEAD: u32 = 1024; // 1 KB - std::mem::size_of::() as u32 - + BLOCK_STUFF_OVERHEAD - + value.data_size().try_into().unwrap_or(u32::MAX) + let mut weight = size_of::() as u32; + + weight = weight.saturating_add(BLOCK_STUFF_OVERHEAD); + weight = weight.saturating_add(value.data_size().try_into().unwrap_or(u32::MAX)); + + // Calculate cells size + let cells_size = value + .num_uniq_cells() + .try_into() + .unwrap_or(u32::MAX) + .saturating_mul(size_of::() as u32); // todo: ask @Rexagon about real size + + weight.saturating_add(cells_size) } let blocks_cache = moka::sync::Cache::builder()