diff --git a/Cargo.lock b/Cargo.lock index 9b7f3bf9d19..6c0b65143cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3123,6 +3123,15 @@ dependencies = [ "similar", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "integration-tests" version = "0.0.0" @@ -3176,6 +3185,7 @@ dependencies = [ "parking_lot 0.12.1", "primitive-types 0.10.1", "rand", + "reed-solomon-erasure", "rlp", "serde", "serde_json", @@ -3298,7 +3308,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" dependencies = [ - "spin", + "spin 0.5.2", ] [[package]] @@ -3340,6 +3350,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "librocksdb-sys" version = "0.11.0+8.1.1" @@ -4465,6 +4481,7 @@ dependencies = [ "rand", "rand_xorshift", "rayon", + "reed-solomon-erasure", "rlimit", "serde", "sha2 0.10.6", @@ -4789,6 +4806,7 @@ dependencies = [ "once_cell", "rand", "rayon", + "reed-solomon-erasure", "rlimit", "rocksdb", "serde", @@ -5734,6 +5752,17 @@ dependencies = [ "parking_lot_core 0.7.2", ] +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api 0.4.7", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -5758,6 +5787,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.2.13", + "smallvec", + "winapi", +] + [[package]] name = "parking_lot_core" version = "0.9.3" @@ -6307,11 +6350,15 @@ dependencies = [ [[package]] name = "reed-solomon-erasure" -version = "4.0.2" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a415a013dd7c5d4221382329a5a3482566da675737494935cbbbcdec04662f9d" +checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f" dependencies = [ + "libm", + "lru 0.7.8", + "parking_lot 0.11.2", "smallvec", + "spin 0.9.8", ] [[package]] @@ -6437,7 +6484,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -7257,6 +7304,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "sptr" version = "0.3.2" diff --git a/Cargo.toml b/Cargo.toml index d8f7c22e067..c62d5760604 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -304,7 +304,7 @@ rand_hc = "0.3.1" rand_xorshift = "0.3" rayon = "1.5" redis = "0.23.0" -reed-solomon-erasure = "4" +reed-solomon-erasure = "6.0.0" regex = "1.7.1" region = "3.0" reqwest = { version = "0.11.14", features = ["blocking"] } diff --git a/chain/chunks/src/lib.rs b/chain/chunks/src/lib.rs index 18cdbff575a..748a0fab42d 100644 --- a/chain/chunks/src/lib.rs +++ b/chain/chunks/src/lib.rs @@ -112,7 +112,7 @@ use near_primitives::errors::EpochError; use near_primitives::hash::CryptoHash; use near_primitives::merkle::{verify_path, MerklePath}; use near_primitives::receipt::Receipt; -use near_primitives::reed_solomon::ReedSolomonWrapper; +use near_primitives::reed_solomon::{reed_solomon_decode, reed_solomon_encode}; use near_primitives::sharding::{ ChunkHash, EncodedShardChunk, EncodedShardChunkBody, PartialEncodedChunk, PartialEncodedChunkPart, PartialEncodedChunkV2, ReceiptProof, ShardChunk, ShardChunkHeader, @@ -129,6 +129,7 @@ use near_primitives::version::ProtocolVersion; use near_primitives::{checked_feature, unwrap_or_return}; use rand::seq::IteratorRandom; use rand::Rng; +use reed_solomon_erasure::galois_8::ReedSolomon; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use tracing::{debug, debug_span, error, warn}; @@ -255,7 +256,7 @@ pub struct ShardsManager { shard_tracker: ShardTracker, peer_manager_adapter: Sender, client_adapter: Sender, - rs: ReedSolomonWrapper, + rs: ReedSolomon, encoded_chunks: EncodedChunksCache, requested_partial_encoded_chunks: RequestPool, @@ -292,10 +293,11 @@ impl ShardsManager { shard_tracker, peer_manager_adapter: network_adapter, client_adapter, - rs: ReedSolomonWrapper::new( + rs: ReedSolomon::new( epoch_manager.num_data_parts(), epoch_manager.num_total_parts() - epoch_manager.num_data_parts(), - ), + ) + .unwrap(), encoded_chunks: EncodedChunksCache::new(), requested_partial_encoded_chunks: RequestPool::new( CHUNK_REQUEST_RETRY, @@ -801,7 +803,7 @@ impl ShardsManager { } pub fn process_partial_encoded_chunk_request( - &mut self, + &self, request: PartialEncodedChunkRequestMsg, route_back: CryptoHash, ) { @@ -837,7 +839,7 @@ impl ShardsManager { /// an explanation of that part of the return value. /// Ensures the receipts in the response are in a deterministic order. fn prepare_partial_encoded_chunk_response( - &mut self, + &self, request: PartialEncodedChunkRequestMsg, ) -> (PartialEncodedChunkResponseSource, PartialEncodedChunkResponseMsg) { let (src, mut response_msg) = self.prepare_partial_encoded_chunk_response_unsorted(request); @@ -848,7 +850,7 @@ impl ShardsManager { } fn prepare_partial_encoded_chunk_response_unsorted( - &mut self, + &self, request: PartialEncodedChunkRequestMsg, ) -> (PartialEncodedChunkResponseSource, PartialEncodedChunkResponseMsg) { let PartialEncodedChunkRequestMsg { chunk_hash, part_ords, mut tracking_shards } = request; @@ -963,8 +965,8 @@ impl ShardsManager { /// expensive operation. If possible, the request should be served from /// EncodedChunksCacheEntry or PartialEncodedChunk instead. // pub for testing - pub fn lookup_partial_encoded_chunk_from_chunk_storage( - &mut self, + fn lookup_partial_encoded_chunk_from_chunk_storage( + &self, part_ords: HashSet, tracking_shards: HashSet, response: &mut PartialEncodedChunkResponseMsg, @@ -990,9 +992,10 @@ impl ShardsManager { // Construct EncodedShardChunk. If we earlier determined that we will // need parity parts, instruct the constructor to calculate them as // well. Otherwise we won’t bother. - let (parts, encoded_length) = self - .rs - .encode(TransactionReceipt(chunk.transactions().to_vec(), outgoing_receipts.to_vec())); + let (parts, encoded_length) = reed_solomon_encode( + &self.rs, + TransactionReceipt(chunk.transactions().to_vec(), outgoing_receipts.to_vec()), + ); if header.encoded_length() != encoded_length as u64 { warn!(target: "chunks", @@ -1051,7 +1054,8 @@ impl ShardsManager { } let encoded_length = chunk.encoded_length(); - if let Err(err) = self.rs.decode::( + if let Err(err) = reed_solomon_decode::( + &self.rs, chunk.content_mut().parts.as_mut_slice(), encoded_length as usize, ) { @@ -1919,7 +1923,7 @@ impl ShardsManager { tx_root: CryptoHash, congestion_info: CongestionInfo, signer: &dyn ValidatorSigner, - rs: &mut ReedSolomonWrapper, + rs: &ReedSolomon, protocol_version: ProtocolVersion, ) -> Result<(EncodedShardChunk, Vec), Error> { EncodedShardChunk::new( @@ -2700,7 +2704,7 @@ mod test { #[test] fn test_chunk_response_for_uncached_partial_chunk() { let mut fixture = ChunkTestFixture::default(); - let mut shards_manager = ShardsManager::new( + let shards_manager = ShardsManager::new( FakeClock::default().clock(), Some(fixture.mock_shard_tracker.clone()), Arc::new(fixture.epoch_manager.clone()), @@ -2732,7 +2736,7 @@ mod test { #[test] fn test_chunk_response_for_uncached_shard_chunk() { let mut fixture = ChunkTestFixture::default(); - let mut shards_manager = ShardsManager::new( + let shards_manager = ShardsManager::new( FakeClock::default().clock(), Some(fixture.mock_shard_tracker.clone()), Arc::new(fixture.epoch_manager.clone()), @@ -2890,7 +2894,7 @@ mod test { #[test] fn test_chunk_response_empty_request() { let fixture = ChunkTestFixture::default(); - let mut shards_manager = ShardsManager::new( + let shards_manager = ShardsManager::new( FakeClock::default().clock(), Some(fixture.mock_shard_tracker.clone()), Arc::new(fixture.epoch_manager.clone()), @@ -2914,7 +2918,7 @@ mod test { #[test] fn test_chunk_response_for_nonexistent_chunk() { let fixture = ChunkTestFixture::default(); - let mut shards_manager = ShardsManager::new( + let shards_manager = ShardsManager::new( FakeClock::default().clock(), Some(fixture.mock_shard_tracker.clone()), Arc::new(fixture.epoch_manager.clone()), @@ -2938,7 +2942,7 @@ mod test { #[test] fn test_chunk_response_for_request_including_invalid_part_ord() { let mut fixture = ChunkTestFixture::default(); - let mut shards_manager = ShardsManager::new( + let shards_manager = ShardsManager::new( FakeClock::default().clock(), Some(fixture.mock_shard_tracker.clone()), Arc::new(fixture.epoch_manager.clone()), @@ -2972,7 +2976,7 @@ mod test { fn test_chunk_response_for_request_with_duplicate_part_ords() { // We should not return any duplicates. let mut fixture = ChunkTestFixture::default(); - let mut shards_manager = ShardsManager::new( + let shards_manager = ShardsManager::new( FakeClock::default().clock(), Some(fixture.mock_shard_tracker.clone()), Arc::new(fixture.epoch_manager.clone()), diff --git a/chain/chunks/src/test/basic.rs b/chain/chunks/src/test/basic.rs index 8bf8d23e164..a58571b0dff 100644 --- a/chain/chunks/src/test/basic.rs +++ b/chain/chunks/src/test/basic.rs @@ -28,7 +28,6 @@ use near_network::{ use near_primitives::types::{AccountId, BlockHeight}; use near_store::test_utils::create_test_store; use std::collections::HashSet; -use tracing::log::info; #[derive(derive_more::AsMut)] struct TestData { @@ -203,7 +202,7 @@ fn test_chunk_forward() { data.chain.record_block(*hash, i as BlockHeight + 1); let next_chunk_producer = data.chain.next_chunk_producer(0); if !chunk_only_producers.contains(&next_chunk_producer) { - info!(target: "test", "Trying again at height {} which has chunk producer {}, we want the next chunk producer to be a chunk only producer", + tracing::info!(target: "test", "Trying again at height {} which has chunk producer {}, we want the next chunk producer to be a chunk only producer", i + 1, next_chunk_producer); continue; } diff --git a/chain/chunks/src/test_loop.rs b/chain/chunks/src/test_loop.rs index 672ab6a2c99..dfa20cecdd7 100644 --- a/chain/chunks/src/test_loop.rs +++ b/chain/chunks/src/test_loop.rs @@ -26,7 +26,6 @@ use near_primitives::congestion_info::CongestionInfo; use near_primitives::{ hash::CryptoHash, merkle::{self, MerklePath}, - reed_solomon::ReedSolomonWrapper, sharding::{ EncodedShardChunk, PartialEncodedChunk, PartialEncodedChunkV2, ReceiptProof, ShardChunkHeader, @@ -36,6 +35,7 @@ use near_primitives::{ version::PROTOCOL_VERSION, }; use near_store::Store; +use reed_solomon_erasure::galois_8::ReedSolomon; use std::{collections::HashMap, sync::Arc}; pub fn forward_client_request_to_shards_manager( @@ -261,7 +261,7 @@ impl MockChainForShardsManager { let signer = create_test_signer(chunk_producer.as_str()); let data_parts = self.epoch_manager.num_data_parts(); let parity_parts = self.epoch_manager.num_total_parts() - data_parts; - let mut rs = ReedSolomonWrapper::new(data_parts, parity_parts); + let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); let (chunk, merkle_paths) = ShardsManager::create_encoded_shard_chunk( self.tip.last_block_hash, CryptoHash::default(), @@ -278,7 +278,7 @@ impl MockChainForShardsManager { MerkleHash::default(), CongestionInfo::default(), &signer, - &mut rs, + &rs, PROTOCOL_VERSION, ) .unwrap(); diff --git a/chain/chunks/src/test_utils.rs b/chain/chunks/src/test_utils.rs index e28b42a2ee2..3b2fa3876ef 100644 --- a/chain/chunks/src/test_utils.rs +++ b/chain/chunks/src/test_utils.rs @@ -10,7 +10,6 @@ use near_primitives::congestion_info::CongestionInfo; use near_primitives::hash::CryptoHash; use near_primitives::merkle::{self, MerklePath}; use near_primitives::receipt::Receipt; -use near_primitives::reed_solomon::ReedSolomonWrapper; use near_primitives::sharding::{ EncodedShardChunk, PartialEncodedChunk, PartialEncodedChunkPart, PartialEncodedChunkV2, ShardChunkHeader, @@ -21,6 +20,7 @@ use near_primitives::types::{AccountId, EpochId, ShardId}; use near_primitives::version::PROTOCOL_VERSION; use near_store::test_utils::create_test_store; use near_store::Store; +use reed_solomon_erasure::galois_8::ReedSolomon; use std::collections::VecDeque; use std::sync::{Arc, Mutex, RwLock}; @@ -47,7 +47,7 @@ pub struct ChunkTestFixture { pub mock_chunk_header: ShardChunkHeader, pub mock_chunk_parts: Vec, pub mock_chain_head: Tip, - pub rs: ReedSolomonWrapper, + pub rs: ReedSolomon, } impl Default for ChunkTestFixture { @@ -87,7 +87,7 @@ impl ChunkTestFixture { let data_parts = epoch_manager.num_data_parts(); let parity_parts = epoch_manager.num_total_parts() - data_parts; - let mut rs = ReedSolomonWrapper::new(data_parts, parity_parts); + let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); let mock_ancestor_hash = CryptoHash::default(); // generate a random block hash for the block at height 1 let (mock_parent_hash, mock_height) = @@ -152,7 +152,7 @@ impl ChunkTestFixture { MerkleHash::default(), CongestionInfo::default(), &signer, - &mut rs, + &rs, PROTOCOL_VERSION, ) .unwrap(); diff --git a/chain/client/src/client.rs b/chain/client/src/client.rs index 9a19ece0481..f0b903690b0 100644 --- a/chain/client/src/client.rs +++ b/chain/client/src/client.rs @@ -69,7 +69,6 @@ use near_primitives::hash::CryptoHash; use near_primitives::merkle::{merklize, MerklePath, PartialMerkleTree}; use near_primitives::network::PeerId; use near_primitives::receipt::Receipt; -use near_primitives::reed_solomon::ReedSolomonWrapper; use near_primitives::sharding::StateSyncInfo; use near_primitives::sharding::{ ChunkHash, EncodedShardChunk, PartialEncodedChunk, ShardChunk, ShardChunkHeader, ShardInfo, @@ -83,6 +82,7 @@ use near_primitives::validator_signer::ValidatorSigner; use near_primitives::version::PROTOCOL_VERSION; use near_primitives::views::{CatchupStatusView, DroppedReason}; use near_store::ShardUId; +use reed_solomon_erasure::galois_8::ReedSolomon; use std::cmp::max; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -159,7 +159,7 @@ pub struct Client { /// List of currently accumulated challenges. pub challenges: HashMap, /// A ReedSolomon instance to reconstruct shard. - pub rs_for_chunk_production: ReedSolomonWrapper, + pub rs_for_chunk_production: ReedSolomon, /// Blocks that have been re-broadcast recently. They should not be broadcast again. rebroadcasted_blocks: lru::LruCache, /// Last time the head was updated, or our head was rebroadcasted. Used to re-broadcast the head @@ -393,7 +393,7 @@ impl Client { block_sync, state_sync, challenges: Default::default(), - rs_for_chunk_production: ReedSolomonWrapper::new(data_parts, parity_parts), + rs_for_chunk_production: ReedSolomon::new(data_parts, parity_parts).unwrap(), rebroadcasted_blocks: lru::LruCache::new(NUM_REBROADCAST_BLOCKS), last_time_head_progress_made: clock.now(), block_production_info: BlockProductionTracker::new(), diff --git a/chain/client/src/stateless_validation/state_witness_actions.rs b/chain/client/src/stateless_validation/state_witness_actions.rs index 78b715e3d89..0d17e03c23b 100644 --- a/chain/client/src/stateless_validation/state_witness_actions.rs +++ b/chain/client/src/stateless_validation/state_witness_actions.rs @@ -7,7 +7,7 @@ use near_async::time::Clock; use near_chain::Error; use near_epoch_manager::EpochManagerAdapter; use near_network::types::{NetworkRequests, PeerManagerAdapter, PeerManagerMessageRequest}; -use near_primitives::reed_solomon::ReedSolomonWrapper; +use near_primitives::reed_solomon::reed_solomon_encode; use near_primitives::sharding::ShardChunkHeader; use near_primitives::stateless_validation::{ ChunkStateWitness, ChunkStateWitnessAck, EncodedChunkStateWitness, PartialEncodedStateWitness, @@ -15,6 +15,7 @@ use near_primitives::stateless_validation::{ }; use near_primitives::types::{AccountId, EpochId}; use near_primitives::validator_signer::ValidatorSigner; +use reed_solomon_erasure::galois_8::ReedSolomon; use crate::metrics; @@ -32,7 +33,7 @@ pub struct StateWitnessActions { state_witness_tracker: ChunkStateWitnessTracker, /// Reed Solomon encoder for encoding state witness parts. /// We keep one wrapper for each length of chunk_validators to avoid re-creating the encoder. - rs_map: HashMap, + rs_map: HashMap, } impl StateWitnessActions { @@ -125,9 +126,9 @@ impl StateWitnessActions { let rs = self.rs_map.entry(chunk_validators.len()).or_insert_with(|| { let total_parts = chunk_validators.len(); let data_parts = std::cmp::max(total_parts * 2 / 3, 1); - ReedSolomonWrapper::new(data_parts, total_parts - data_parts) + ReedSolomon::new(data_parts, total_parts - data_parts).unwrap() }); - let (parts, encoded_length) = rs.encode(witness_bytes); + let (parts, encoded_length) = reed_solomon_encode(&rs, witness_bytes); let validator_witness_tuple = chunk_validators .iter() diff --git a/chain/client/src/test_utils/client.rs b/chain/client/src/test_utils/client.rs index 75b52252369..505a6ef9342 100644 --- a/chain/client/src/test_utils/client.rs +++ b/chain/client/src/test_utils/client.rs @@ -20,7 +20,6 @@ use near_network::types::HighestHeightPeerInfo; use near_primitives::block::Block; use near_primitives::hash::CryptoHash; use near_primitives::merkle::{merklize, PartialMerkleTree}; -use near_primitives::reed_solomon::ReedSolomonWrapper; use near_primitives::sharding::{EncodedShardChunk, ShardChunk}; use near_primitives::stateless_validation::ChunkEndorsement; use near_primitives::transaction::SignedTransaction; @@ -28,6 +27,7 @@ use near_primitives::types::{BlockHeight, ShardId}; use near_primitives::utils::MaybeValidated; use near_primitives::version::PROTOCOL_VERSION; use num_rational::Ratio; +use reed_solomon_erasure::galois_8::ReedSolomon; impl Client { /// Unlike Client::start_process_block, which returns before the block finishes processing @@ -188,7 +188,7 @@ pub fn create_chunk( let data_parts = client.chain.epoch_manager.num_data_parts(); let decoded_chunk = chunk.decode_chunk(data_parts).unwrap(); let parity_parts = total_parts - data_parts; - let mut rs = ReedSolomonWrapper::new(data_parts, parity_parts); + let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); let signer = client.validator_signer.as_ref().unwrap().clone(); let header = chunk.cloned_header(); @@ -198,7 +198,7 @@ pub fn create_chunk( header.prev_outcome_root(), header.height_created(), header.shard_id(), - &mut rs, + &rs, header.prev_gas_used(), header.gas_limit(), header.prev_balance_burnt(), diff --git a/chain/network/Cargo.toml b/chain/network/Cargo.toml index c7e301fa148..3d28bcb5254 100644 --- a/chain/network/Cargo.toml +++ b/chain/network/Cargo.toml @@ -38,6 +38,7 @@ pin-project.workspace = true protobuf.workspace = true rand.workspace = true rayon.workspace = true +reed-solomon-erasure.workspace = true serde.workspace = true smart-default.workspace = true sha2.workspace = true diff --git a/chain/network/src/network_protocol/testonly.rs b/chain/network/src/network_protocol/testonly.rs index f25f8872ae5..44b9435005a 100644 --- a/chain/network/src/network_protocol/testonly.rs +++ b/chain/network/src/network_protocol/testonly.rs @@ -13,7 +13,7 @@ use near_primitives::challenge::{BlockDoubleSign, Challenge, ChallengeBody}; use near_primitives::hash::CryptoHash; use near_primitives::network::{AnnounceAccount, PeerId}; use near_primitives::num_rational::Ratio; -use near_primitives::reed_solomon::ReedSolomonWrapper; +use near_primitives::reed_solomon::reed_solomon_encode; use near_primitives::sharding::{ ChunkHash, EncodedShardChunkBody, PartialEncodedChunkPart, ShardChunk, }; @@ -23,6 +23,7 @@ use near_primitives::validator_signer::{InMemoryValidatorSigner, ValidatorSigner use near_primitives::version; use rand::distributions::Standard; use rand::Rng; +use reed_solomon_erasure::galois_8::ReedSolomon; use std::collections::HashMap; use std::net; use std::sync::Arc; @@ -182,10 +183,10 @@ pub fn make_challenge(rng: &mut R) -> Challenge { pub fn make_chunk_parts(chunk: ShardChunk) -> Vec { let total_shard_count = 10; let parity_shard_count = 5; - let mut rs = ReedSolomonWrapper::new(total_shard_count, parity_shard_count); + let rs = ReedSolomon::new(total_shard_count, parity_shard_count).unwrap(); let transaction_receipts = (chunk.transactions().to_vec(), chunk.prev_outgoing_receipts().to_vec()); - let (parts, _) = rs.encode(transaction_receipts); + let (parts, _) = reed_solomon_encode(&rs, transaction_receipts); let mut content = EncodedShardChunkBody { parts }; let (_, merkle_paths) = content.get_merkle_hash_and_paths(); diff --git a/core/primitives/src/block.rs b/core/primitives/src/block.rs index 366e92fbc1f..f9f82e26c12 100644 --- a/core/primitives/src/block.rs +++ b/core/primitives/src/block.rs @@ -10,7 +10,6 @@ use crate::congestion_info::CongestionInfo; use crate::hash::{hash, CryptoHash}; use crate::merkle::{merklize, verify_path, MerklePath}; use crate::num_rational::Rational32; -use crate::reed_solomon::ReedSolomonWrapper; use crate::sharding::{ ChunkHashHeight, EncodedShardChunk, ShardChunk, ShardChunkHeader, ShardChunkHeaderV1, }; @@ -22,6 +21,7 @@ use near_async::time::Utc; use near_crypto::Signature; use near_primitives_core::types::ShardId; use primitive_types::U256; +use reed_solomon_erasure::galois_8::ReedSolomon; use std::collections::HashMap; use std::ops::Index; use std::sync::Arc; @@ -97,7 +97,7 @@ pub fn genesis_chunks( genesis_height: BlockHeight, genesis_protocol_version: ProtocolVersion, ) -> Vec { - let mut rs = ReedSolomonWrapper::new(1, 2); + let rs = ReedSolomon::new(1, 2).unwrap(); let state_roots = if state_roots.len() == shard_ids.len() { state_roots } else { @@ -115,7 +115,7 @@ pub fn genesis_chunks( CryptoHash::default(), genesis_height, shard_id, - &mut rs, + &rs, 0, initial_gas_limit, 0, diff --git a/core/primitives/src/reed_solomon.rs b/core/primitives/src/reed_solomon.rs index ec3af9448af..8aea8cf4710 100644 --- a/core/primitives/src/reed_solomon.rs +++ b/core/primitives/src/reed_solomon.rs @@ -1,85 +1,54 @@ use borsh::{BorshDeserialize, BorshSerialize}; use itertools::Itertools; -use reed_solomon_erasure::galois_8::{Field, ReedSolomon}; -use reed_solomon_erasure::ReconstructShard; +use reed_solomon_erasure::galois_8::ReedSolomon; use std::io::Error; -/// The ttl for a reed solomon instance to control memory usage. This number below corresponds to -/// roughly 60MB of memory usage. -const RS_TTL: u64 = 2 * 1024; - -/// Wrapper around reed solomon which occasionally resets the underlying -/// reed solomon instead to work around the memory leak in reed solomon -/// implementation -pub struct ReedSolomonWrapper { - rs: ReedSolomon, - ttl: u64, +// Encode function takes a serializable object and returns a tuple of parts and length of encoded data +pub fn reed_solomon_encode( + rs: &ReedSolomon, + data: T, +) -> (Vec>>, usize) { + let mut bytes = borsh::to_vec(&data).unwrap(); + let encoded_length = bytes.len(); + + let data_parts = rs.data_shard_count(); + let part_length = (encoded_length + data_parts - 1) / data_parts; + + // Pad the bytes to be a multiple of `part_length` + // Convert encoded data into `data_shard_count` number of parts and pad with `parity_shard_count` None values + // with 4 data_parts and 2 parity_parts + // b'aaabbbcccd' -> [Some(b'aaa'), Some(b'bbb'), Some(b'ccc'), Some(b'd00'), None, None] + bytes.resize(data_parts * part_length, 0); + let mut parts = bytes + .chunks_exact(part_length) + .map(|chunk| Some(chunk.to_vec().into_boxed_slice())) + .chain(itertools::repeat_n(None, rs.parity_shard_count())) + .collect_vec(); + + // Fine to unwrap here as we just constructed the parts + rs.reconstruct(&mut parts).unwrap(); + + (parts, encoded_length) } -impl ReedSolomonWrapper { - pub fn new(data_shards: usize, parity_shards: usize) -> Self { - ReedSolomonWrapper { - rs: ReedSolomon::new(data_shards, parity_shards).unwrap(), - ttl: RS_TTL, - } +// Decode function is the reverse of encode function. It takes parts and length of encoded data +// and returns the deserialized object. +// Return an error if the reed solomon decoding fails or borsh deserialization fails. +pub fn reed_solomon_decode( + rs: &ReedSolomon, + parts: &mut [Option>], + encoded_length: usize, +) -> Result { + if let Err(err) = rs.reconstruct(parts) { + return Err(Error::other(err)); } - // Encode function takes a serializable object and returns a tuple of parts and length of encoded data - pub fn encode(&mut self, data: T) -> (Vec>>, usize) { - let mut bytes = borsh::to_vec(&data).unwrap(); - let encoded_length = bytes.len(); - - let data_parts = self.rs.data_shard_count(); - let part_length = (encoded_length + data_parts - 1) / data_parts; - - // Pad the bytes to be a multiple of `part_length` - // Convert encoded data into `data_shard_count` number of parts and pad with `parity_shard_count` None values - // with 4 data_parts and 2 parity_parts - // b'aaabbbcccd' -> [Some(b'aaa'), Some(b'bbb'), Some(b'ccc'), Some(b'd00'), None, None] - bytes.resize(data_parts * part_length, 0); - let mut parts = bytes - .chunks_exact(part_length) - .map(|chunk| Some(chunk.to_vec().into_boxed_slice())) - .chain(itertools::repeat_n(None, self.rs.parity_shard_count())) - .collect_vec(); - - // Fine to unwrap here as we just constructed the parts - self.reconstruct(&mut parts).unwrap(); - - (parts, encoded_length) - } + let encoded_data = parts + .iter() + .flat_map(|option| option.as_ref().expect("Missing shard").iter()) + .cloned() + .take(encoded_length) + .collect_vec(); - // Decode function is the reverse of encode function. It takes parts and length of encoded data - // and returns the deserialized object. - // Return an error if the reed solomon decoding fails or borsh deserialization fails. - pub fn decode( - &mut self, - parts: &mut [Option>], - encoded_length: usize, - ) -> Result { - if let Err(err) = self.reconstruct(parts) { - return Err(Error::other(err)); - } - - let encoded_data = parts - .iter() - .flat_map(|option| option.as_ref().expect("Missing shard").iter()) - .cloned() - .take(encoded_length) - .collect_vec(); - - T::try_from_slice(&encoded_data) - } - - fn reconstruct>( - &mut self, - slices: &mut [T], - ) -> Result<(), reed_solomon_erasure::Error> { - self.ttl -= 1; - if self.ttl == 0 { - *self = - ReedSolomonWrapper::new(self.rs.data_shard_count(), self.rs.parity_shard_count()); - } - self.rs.reconstruct(slices) - } + T::try_from_slice(&encoded_data) } diff --git a/core/primitives/src/sharding.rs b/core/primitives/src/sharding.rs index 2192c696ae0..42e7bed6380 100644 --- a/core/primitives/src/sharding.rs +++ b/core/primitives/src/sharding.rs @@ -2,7 +2,7 @@ use crate::congestion_info::CongestionInfo; use crate::hash::{hash, CryptoHash}; use crate::merkle::{combine_hash, merklize, verify_path, MerklePath}; use crate::receipt::Receipt; -use crate::reed_solomon::ReedSolomonWrapper; +use crate::reed_solomon::reed_solomon_encode; use crate::transaction::SignedTransaction; use crate::types::validator_stake::{ValidatorStake, ValidatorStakeIter, ValidatorStakeV1}; use crate::types::{Balance, BlockHeight, Gas, MerkleHash, ShardId, StateRoot}; @@ -11,6 +11,7 @@ use crate::version::{ProtocolFeature, ProtocolVersion, SHARD_CHUNK_HEADER_UPGRAD use borsh::{BorshDeserialize, BorshSerialize}; use near_crypto::Signature; use near_fmt::AbbrBytes; +use reed_solomon_erasure::galois_8::ReedSolomon; use std::cmp::Ordering; use std::sync::Arc; use tracing::debug_span; @@ -1020,7 +1021,7 @@ impl EncodedShardChunk { prev_outcome_root: CryptoHash, height: BlockHeight, shard_id: ShardId, - rs: &mut ReedSolomonWrapper, + rs: &ReedSolomon, prev_gas_used: Gas, gas_limit: Gas, prev_balance_burnt: Balance, @@ -1033,8 +1034,10 @@ impl EncodedShardChunk { signer: &dyn ValidatorSigner, protocol_version: ProtocolVersion, ) -> Result<(Self, Vec), std::io::Error> { - let (transaction_receipts_parts, encoded_length) = - rs.encode(TransactionReceipt(transactions, prev_outgoing_receipts.to_vec())); + let (transaction_receipts_parts, encoded_length) = reed_solomon_encode( + rs, + TransactionReceipt(transactions, prev_outgoing_receipts.to_vec()), + ); let content = EncodedShardChunkBody { parts: transaction_receipts_parts }; let (encoded_merkle_root, merkle_paths) = content.get_merkle_hash_and_paths(); diff --git a/core/store/Cargo.toml b/core/store/Cargo.toml index 4d4d4c8a311..c6aa9721c86 100644 --- a/core/store/Cargo.toml +++ b/core/store/Cargo.toml @@ -29,6 +29,7 @@ num_cpus.workspace = true once_cell.workspace = true rand.workspace = true rayon.workspace = true +reed-solomon-erasure.workspace = true rlimit.workspace = true rocksdb.workspace = true serde.workspace = true diff --git a/core/store/benches/finalize_bench.rs b/core/store/benches/finalize_bench.rs index f5a30b7f848..673d6227bd1 100644 --- a/core/store/benches/finalize_bench.rs +++ b/core/store/benches/finalize_bench.rs @@ -23,7 +23,6 @@ use near_primitives::congestion_info::CongestionInfo; use near_primitives::hash::CryptoHash; use near_primitives::merkle::{merklize, MerklePathItem}; use near_primitives::receipt::{ActionReceipt, DataReceipt, Receipt, ReceiptEnum}; -use near_primitives::reed_solomon::ReedSolomonWrapper; use near_primitives::shard_layout::ShardLayout; use near_primitives::sharding::{ ChunkHash, EncodedShardChunk, PartialEncodedChunk, PartialEncodedChunkPart, @@ -36,6 +35,7 @@ use near_primitives::validator_signer::InMemoryValidatorSigner; use near_primitives::version::PROTOCOL_VERSION; use near_store::DBCol; use rand::prelude::SliceRandom; +use reed_solomon_erasure::galois_8::ReedSolomon; /// `ShardChunk` -> `StoreUpdate::insert_ser`. /// @@ -183,7 +183,7 @@ fn create_encoded_shard_chunk( transactions: Vec, receipts: &[Receipt], ) -> (EncodedShardChunk, Vec>) { - let mut rs = ReedSolomonWrapper::new(33, 67); + let rs = ReedSolomon::new(33, 67).unwrap(); ShardsManager::create_encoded_shard_chunk( Default::default(), Default::default(), @@ -200,7 +200,7 @@ fn create_encoded_shard_chunk( Default::default(), CongestionInfo::default(), &validator_signer(), - &mut rs, + &rs, 100, ) .unwrap() diff --git a/deny.toml b/deny.toml index 83e1fe19117..38b454a247b 100644 --- a/deny.toml +++ b/deny.toml @@ -157,4 +157,9 @@ skip = [ # Everything depends on this... { name = "lru", version = "=0.7.8" }, + + # reed-solomon-erasure latest version + { name = "parking_lot", version = "=0.11.2" }, + { name = "parking_lot_core", version = "=0.8.6" }, + { name = "spin", version = "=0.9.8" }, ] diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 2f3606f20ce..e0b34d6af37 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -27,6 +27,7 @@ once_cell.workspace = true parking_lot.workspace = true primitive-types.workspace = true rand.workspace = true +reed-solomon-erasure.workspace = true serde.workspace = true serde_json.workspace = true smart-default.workspace = true diff --git a/integration-tests/src/tests/client/challenges.rs b/integration-tests/src/tests/client/challenges.rs index eaef7092c35..043710cd850 100644 --- a/integration-tests/src/tests/client/challenges.rs +++ b/integration-tests/src/tests/client/challenges.rs @@ -16,7 +16,6 @@ use near_primitives::congestion_info::CongestionInfo; use near_primitives::hash::CryptoHash; use near_primitives::merkle::PartialMerkleTree; use near_primitives::num_rational::Ratio; -use near_primitives::reed_solomon::ReedSolomonWrapper; use near_primitives::shard_layout::ShardUId; use near_primitives::sharding::EncodedShardChunk; use near_primitives::stateless_validation::ChunkEndorsement; @@ -28,6 +27,7 @@ use near_primitives::validator_signer::ValidatorSigner; use near_primitives::version::PROTOCOL_VERSION; use near_store::Trie; use nearcore::test_utils::TestEnvNightshadeSetupExt; +use reed_solomon_erasure::galois_8::ReedSolomon; /// Check that block containing a challenge is rejected. /// TODO (#2445): Enable challenges when they are working correctly. @@ -361,7 +361,7 @@ fn test_verify_chunk_invalid_state_challenge() { let total_parts = env.clients[0].epoch_manager.num_total_parts(); let data_parts = env.clients[0].epoch_manager.num_data_parts(); let parity_parts = total_parts - data_parts; - let mut rs = ReedSolomonWrapper::new(data_parts, parity_parts); + let rs = ReedSolomon::new(data_parts, parity_parts).unwrap(); let (mut invalid_chunk, merkle_paths) = ShardsManager::create_encoded_shard_chunk( *last_block.hash(), Trie::EMPTY_ROOT, @@ -378,7 +378,7 @@ fn test_verify_chunk_invalid_state_challenge() { CryptoHash::default(), CongestionInfo::default(), &validator_signer, - &mut rs, + &rs, PROTOCOL_VERSION, ) .unwrap();