Merge remote-tracking branch 'origin/master' into nep-491_reference_i…

…mplementation

.github/workflows/neard_release.yml

@@ -39,6 +39,8 @@ jobs:
       - name: Checkout nearcore repository
         if: ${{ github.event_name != 'workflow_dispatch'}}
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Neard binary build and upload to S3
         run: ./scripts/binary_release.sh
@@ -67,6 +69,8 @@ jobs:
       - name: Checkout nearcore repository
         if: ${{ github.event_name != 'workflow_dispatch'}}
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3
@@ -77,7 +81,11 @@ jobs:
       - name: Build and push Docker image to Dockerhub
         run: |
           COMMIT=$(git rev-parse HEAD)
-          BRANCH=${{ github.ref_name }}
+          BRANCH=$(git branch --show-current)
+          # in case of Release triggered run, branch is empty
+          if [ -z "$BRANCH" ]; then
+            BRANCH=$(git branch -r --contains=${{ github.ref_name }} | head -n1 | cut -c3- | cut -d / -f 2)
+          fi
           make docker-nearcore
           docker tag nearcore nearprotocol/nearcore:${BRANCH}-${COMMIT}
           docker tag nearcore nearprotocol/nearcore:${BRANCH}

Justfile

@@ -92,13 +92,30 @@ codecov RULE:
     # Note: macos seems to not support `source <()` as a way to set environment variables, but
     # this variant seems to work on both linux and macos.
     # TODO: remove the RUSTFLAGS hack, see also https://github.com/rust-lang/cargo/issues/13040
-    cargo llvm-cov show-env --export-prefix | grep -v RUSTFLAGS > env
+    cargo llvm-cov show-env --export-prefix | grep -v RUSTFLAGS= > env
     source ./env
-    export RUSTC_WORKSPACE_WRAPPER="{{ absolute_path("scripts/rustc-coverage-wrapper.sh") }}"
+    export RUSTC_WORKSPACE_WRAPPER="{{ absolute_path("scripts/coverage-wrapper-rustc") }}"
     {{ just_executable() }} {{ RULE }}
-    cargo llvm-cov report --profile dev-release --codecov --output-path codecov.json
-    # See https://github.com/taiki-e/cargo-llvm-cov/issues/292
-    find target -name '*.profraw' -delete
+    mkdir -p coverage/codecov
+    cargo llvm-cov report --profile dev-release --codecov --output-path coverage/codecov/new.json
+
+# generate a codecov report for RULE, CI version
+codecov-ci RULE:
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    {{ just_executable() }} codecov "{{ RULE }}"
+    pushd target
+    tar -c --zstd -f ../coverage/profraw/new.tar.zst *.profraw
+    popd
+    rm -rf target/*.profraw
+
+# generate a tarball with all the binaries for coverage CI
+tar-bins-for-coverage-ci:
+    #!/usr/bin/env bash
+    find target/dev-release/ \( -name incremental -or -name .fingerprint -or -name out \) -exec rm -rf '{}' \; || true
+    find target/dev-release/ -not -executable -delete || true
+    find target/dev-release/ -name 'build*script*build*' -delete || true
+    tar -c --zstd -f coverage/profraw/binaries/new.tar.zst target/dev-release/
 
 # style checks from python scripts
 python-style-checks:

chain/chain/src/chain.rs

@@ -1265,7 +1265,7 @@ impl Chain {
             }
             let partial_encoded_chunk =
                 self.chain_store.get_partial_chunk(&chunk_header.chunk_hash()).unwrap();
-            for receipt in partial_encoded_chunk.receipts().iter() {
+            for receipt in partial_encoded_chunk.prev_outgoing_receipts().iter() {
                 let ReceiptProof(_, shard_proof) = receipt;
                 let ShardProof { to_shard_id, .. } = shard_proof;
                 receipt_proofs_by_shard_id
@@ -2014,7 +2014,7 @@ impl Chain {
 
         self.validate_chunk_headers(&block, &prev_block)?;
 
-        if checked_feature!("stable", ChunkValidation, protocol_version) {
+        if checked_feature!("stable", StatelessValidationV0, protocol_version) {
             self.validate_chunk_endorsements_in_block(&block)?;
         }
 
@@ -2873,6 +2873,21 @@ impl Chain {
         Ok(())
     }
 
+    pub fn transaction_validity_check<'a>(
+        &'a self,
+        prev_block_header: BlockHeader,
+    ) -> impl FnMut(&SignedTransaction) -> bool + 'a {
+        move |tx: &SignedTransaction| -> bool {
+            self.chain_store()
+                .check_transaction_validity_period(
+                    &prev_block_header,
+                    &tx.transaction.block_hash,
+                    self.transaction_validity_period,
+                )
+                .is_ok()
+        }
+    }
+
     /// For given pair of block headers and shard id, return information about
     /// block necessary for processing shard update.
     pub fn get_apply_chunk_block_context(
@@ -3230,7 +3245,7 @@ impl Chain {
             self.epoch_manager.get_next_epoch_id_from_prev_block(block_header.prev_hash())?;
         let next_protocol_version =
             self.epoch_manager.get_epoch_protocol_version(&next_epoch_id)?;
-        if !checked_feature!("stable", ChunkValidation, next_protocol_version) {
+        if !checked_feature!("stable", StatelessValidationV0, next_protocol_version) {
             // Chunk validation not enabled yet.
             return Ok(false);
         }

chain/chain/src/chunk_endorsement.rs

@@ -2,7 +2,7 @@ use std::collections::HashSet;
 
 use near_chain_primitives::Error;
 use near_primitives::block::Block;
-use near_primitives::chunk_validation::ChunkEndorsement;
+use near_primitives::stateless_validation::ChunkEndorsement;
 
 use crate::Chain;
 
@@ -68,12 +68,12 @@ impl Chain {
             // Signature can be either None, or Some(signature).
             // We calculate the stake of the chunk_validators for who we have the signature present.
             let mut endorsed_chunk_validators = HashSet::new();
-            for (account_id, signature) in ordered_chunk_validators.into_iter().zip(signatures) {
+            for (account_id, signature) in ordered_chunk_validators.iter().zip(signatures) {
                 let Some(signature) = signature else { continue };
                 let (validator, _) = self.epoch_manager.get_validator_by_account_id(
                     &epoch_id,
                     block.header().prev_hash(),
-                    &account_id,
+                    account_id,
                 )?;
 
                 // Block should not be produced with an invalid signature.
@@ -95,9 +95,10 @@ impl Chain {
                 endorsed_chunk_validators.insert(account_id);
             }
 
-            if !chunk_validator_assignments.does_chunk_have_enough_stake(&endorsed_chunk_validators)
-            {
-                tracing::error!(target: "chain", "Chunk does not have enough stake to be endorsed");
+            let endorsement_stats =
+                chunk_validator_assignments.compute_endorsement_stats(&endorsed_chunk_validators);
+            if !endorsement_stats.has_enough_stake() {
+                tracing::error!(target: "chain", ?endorsement_stats, "Chunk does not have enough stake to be endorsed");
                 return Err(Error::InvalidChunkEndorsement);
             }
         }

chain/chain/src/garbage_collection.rs

@@ -525,7 +525,7 @@ impl<'a> ChainStoreUpdate<'a> {
     ) -> Result<(), Error> {
         let mut store_update = self.store().store_update();
 
-        tracing::info!(target: "garbage_collection", ?gc_mode, ?block_hash, "GC block_hash");
+        tracing::debug!(target: "garbage_collection", ?gc_mode, ?block_hash, "GC block_hash");
 
         // 1. Apply revert insertions or deletions from DBCol::TrieChanges for Trie
         {
@@ -586,6 +586,7 @@ impl<'a> ChainStoreUpdate<'a> {
             let block_shard_id = get_block_shard_id(&block_hash, shard_id);
             self.gc_outgoing_receipts(&block_hash, shard_id);
             self.gc_col(DBCol::IncomingReceipts, &block_shard_id);
+            self.gc_col(DBCol::StateTransitionData, &block_shard_id);
 
             // For incoming State Parts it's done in chain.clear_downloaded_parts()
             // The following code is mostly for outgoing State Parts.
@@ -687,6 +688,8 @@ impl<'a> ChainStoreUpdate<'a> {
             self.gc_outgoing_receipts(&block_hash, shard_id);
             self.gc_col(DBCol::IncomingReceipts, &block_shard_id);
 
+            self.gc_col(DBCol::StateTransitionData, &block_shard_id);
+
             // delete DBCol::ChunkExtra based on shard_uid since it's indexed by shard_uid in the storage
             self.gc_col(DBCol::ChunkExtra, &block_shard_id);
 

chain/chain/src/store.rs

@@ -50,7 +50,7 @@ use near_store::{
 use crate::byzantine_assert;
 use crate::chunks_store::ReadOnlyChunksStore;
 use crate::types::{Block, BlockHeader, LatestKnown};
-use near_primitives::chunk_validation::StoredChunkStateTransitionData;
+use near_primitives::stateless_validation::StoredChunkStateTransitionData;
 use near_store::db::{StoreStatistics, STATE_SYNC_DUMP_KEY};
 use std::sync::Arc;
 

chain/chain/src/test_utils.rs

@@ -307,7 +307,7 @@ pub fn display_chain(me: &Option<AccountId>, chain: &mut Chain, tail: bool) {
                             chunk_producer,
                             partial_chunk.parts().iter().map(|x| x.part_ord).collect::<Vec<_>>(),
                             partial_chunk
-                                .receipts()
+                                .prev_outgoing_receipts()
                                 .iter()
                                 .map(|x| format!("{} => {}", x.0.len(), x.1.to_shard_id))
                                 .collect::<Vec<_>>(),

chain/chain/src/test_utils/kv_runtime.rs

@@ -1,7 +1,8 @@
 use super::ValidatorSchedule;
 use crate::types::{
     ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, ApplyResultForResharding,
-    PreparedTransactions, RuntimeAdapter, RuntimeStorageConfig,
+    PrepareTransactionsBlockContext, PrepareTransactionsChunkContext, PreparedTransactions,
+    RuntimeAdapter, RuntimeStorageConfig,
 };
 use crate::BlockHeader;
 use borsh::{BorshDeserialize, BorshSerialize};
@@ -13,9 +14,6 @@ use near_epoch_manager::{EpochManagerAdapter, RngSeed};
 use near_pool::types::TransactionGroupIterator;
 use near_primitives::account::{AccessKey, Account};
 use near_primitives::block_header::{Approval, ApprovalInner};
-use near_primitives::chunk_validation::{
-    ChunkEndorsement, ChunkStateWitness, ChunkValidatorAssignments,
-};
 use near_primitives::epoch_manager::block_info::BlockInfo;
 use near_primitives::epoch_manager::epoch_info::EpochInfo;
 use near_primitives::epoch_manager::EpochConfig;
@@ -28,13 +26,16 @@ use near_primitives::shard_layout;
 use near_primitives::shard_layout::{ShardLayout, ShardUId};
 use near_primitives::sharding::{ChunkHash, ShardChunkHeader};
 use near_primitives::state_part::PartId;
+use near_primitives::stateless_validation::{
+    ChunkEndorsement, ChunkStateWitness, ChunkValidatorAssignments,
+};
 use near_primitives::transaction::{
     Action, ExecutionMetadata, ExecutionOutcome, ExecutionOutcomeWithId, ExecutionStatus,
     SignedTransaction, TransferAction,
 };
 use near_primitives::types::validator_stake::ValidatorStake;
 use near_primitives::types::{
-    AccountId, ApprovalStake, Balance, BlockHeight, EpochHeight, EpochId, Gas, Nonce, NumShards,
+    AccountId, ApprovalStake, Balance, BlockHeight, EpochHeight, EpochId, Nonce, NumShards,
     ShardId, StateChangesForResharding, StateRoot, StateRootNode, ValidatorInfoIdentifier,
 };
 use near_primitives::version::{ProtocolVersion, PROTOCOL_VERSION};
@@ -1045,15 +1046,11 @@ impl RuntimeAdapter for KeyValueRuntime {
 
     fn prepare_transactions(
         &self,
-        _gas_price: Balance,
-        _gas_limit: Gas,
-        _epoch_id: &EpochId,
-        _shard_id: ShardId,
-        _storage_config: RuntimeStorageConfig,
-        _next_block_height: BlockHeight,
+        _storage: RuntimeStorageConfig,
+        _chunk: PrepareTransactionsChunkContext,
+        _prev_block: PrepareTransactionsBlockContext,
         transaction_groups: &mut dyn TransactionGroupIterator,
         _chain_validate: &mut dyn FnMut(&SignedTransaction) -> bool,
-        _current_protocol_version: ProtocolVersion,
         _time_limit: Option<Duration>,
     ) -> Result<PreparedTransactions, Error> {
         let mut res = vec![];

chain/chain/src/types.rs

@@ -8,6 +8,7 @@ use near_chain_configs::MutableConfigValue;
 use near_chain_configs::ReshardingConfig;
 use near_pool::types::TransactionGroupIterator;
 use near_primitives::sandbox::state_patch::SandboxStatePatch;
+use near_primitives::sharding::ShardChunkHeader;
 use near_store::flat::FlatStorageManager;
 use near_store::StorageError;
 use num_rational::Rational32;
@@ -335,6 +336,32 @@ pub enum PrepareTransactionsLimit {
     ReceiptCount,
 }
 
+pub struct PrepareTransactionsBlockContext {
+    pub next_gas_price: Balance,
+    pub height: BlockHeight,
+    pub block_hash: CryptoHash,
+}
+
+impl From<&BlockHeader> for PrepareTransactionsBlockContext {
+    fn from(header: &BlockHeader) -> Self {
+        Self {
+            next_gas_price: header.next_gas_price(),
+            height: header.height(),
+            block_hash: *header.hash(),
+        }
+    }
+}
+pub struct PrepareTransactionsChunkContext {
+    pub shard_id: ShardId,
+    pub gas_limit: Gas,
+}
+
+impl From<&ShardChunkHeader> for PrepareTransactionsChunkContext {
+    fn from(header: &ShardChunkHeader) -> Self {
+        Self { shard_id: header.shard_id(), gas_limit: header.gas_limit() }
+    }
+}
+
 /// Bridge between the chain and the runtime.
 /// Main function is to update state given transactions.
 /// Additionally handles validators.
@@ -392,15 +419,11 @@ pub trait RuntimeAdapter: Send + Sync {
     /// `RuntimeError::StorageError`.
     fn prepare_transactions(
         &self,
-        gas_price: Balance,
-        gas_limit: Gas,
-        epoch_id: &EpochId,
-        shard_id: ShardId,
         storage: RuntimeStorageConfig,
-        next_block_height: BlockHeight,
+        chunk: PrepareTransactionsChunkContext,
+        prev_block: PrepareTransactionsBlockContext,
         transaction_groups: &mut dyn TransactionGroupIterator,
         chain_validate: &mut dyn FnMut(&SignedTransaction) -> bool,
-        current_protocol_version: ProtocolVersion,
         time_limit: Option<Duration>,
     ) -> Result<PreparedTransactions, Error>;
 

chain/chunks/src/chunk_cache.rs

@@ -94,7 +94,7 @@ impl EncodedChunksCacheEntry {
             });
         }
 
-        for receipt in partial_encoded_chunk.receipts.iter() {
+        for receipt in partial_encoded_chunk.prev_outgoing_receipts.iter() {
             let shard_id = receipt.1.to_shard_id;
             self.receipts.entry(shard_id).or_insert_with(|| receipt.clone());
         }
@@ -309,7 +309,7 @@ mod tests {
         cache.merge_in_partial_encoded_chunk(&PartialEncodedChunkV2 {
             header: header1.clone(),
             parts: vec![],
-            receipts: vec![],
+            prev_outgoing_receipts: vec![],
         });
         assert_eq!(
             cache.get_incomplete_chunks(&CryptoHash::default()).unwrap(),
@@ -329,7 +329,7 @@ mod tests {
         let mut cache = EncodedChunksCache::new();
         let header = create_chunk_header(1, 0);
         let partial_encoded_chunk =
-            PartialEncodedChunkV2 { header: header, parts: vec![], receipts: vec![] };
+            PartialEncodedChunkV2 { header: header, parts: vec![], prev_outgoing_receipts: vec![] };
         cache.merge_in_partial_encoded_chunk(&partial_encoded_chunk);
         assert!(!cache.height_map.is_empty());
 

chain/chunks/src/lib.rs

@@ -930,7 +930,7 @@ impl ShardsManager {
                 response.parts.push(part.clone());
             }
         }
-        for receipt in partial_chunk.receipts() {
+        for receipt in partial_chunk.prev_outgoing_receipts() {
             if tracking_shards.contains(&receipt.1.to_shard_id) {
                 response.receipts.push(receipt.clone());
             }
@@ -1241,7 +1241,7 @@ impl ShardsManager {
         let partial_chunk = PartialEncodedChunk::V2(PartialEncodedChunkV2 {
             header,
             parts: forward.parts,
-            receipts: Vec::new(),
+            prev_outgoing_receipts: Vec::new(),
         });
         self.process_partial_encoded_chunk(MaybeValidated::from_validated(partial_chunk))?;
         Ok(())
@@ -1358,7 +1358,7 @@ impl ShardsManager {
             self.encoded_chunks.merge_in_partial_encoded_chunk(&PartialEncodedChunkV2 {
                 header: header.clone(),
                 parts: parts.into_values().collect(),
-                receipts: vec![],
+                prev_outgoing_receipts: vec![],
             });
             return true;
         }
@@ -1464,7 +1464,7 @@ impl ShardsManager {
         }
 
         // 1.e Checking receipts validity
-        for proof in partial_encoded_chunk.receipts.iter() {
+        for proof in partial_encoded_chunk.prev_outgoing_receipts.iter() {
             // TODO: only validate receipts we care about
             // https://github.com/near/nearcore/issues/5885
             // we can't simply use prev_block_hash to check if the node tracks this shard or not
@@ -2466,7 +2466,7 @@ mod test {
         let partial_encoded_chunk = PartialEncodedChunk::V2(PartialEncodedChunkV2 {
             header: fixture.mock_chunk_header.clone(),
             parts: other_parts,
-            receipts: Vec::new(),
+            prev_outgoing_receipts: Vec::new(),
         });
         // The validator receives a chunk header with the rest of the parts it needed
         let result = shards_manager