Merge branch 'master' into tier3

.buildkite/pipeline.yml

@@ -32,7 +32,6 @@ steps:
 
       python3 scripts/check_nightly.py
       python3 scripts/check_pytests.py
-      python3 scripts/check_fuzzing.py
 
       python3 scripts/fix_nightly_feature_flags.py
 

.github/workflows/ci.yml

@@ -130,7 +130,6 @@ jobs:
 
       - run: python3 scripts/check_nightly.py
       - run: python3 scripts/check_pytests.py
-      - run: python3 scripts/check_fuzzing.py
 
       - run: python3 scripts/fix_nightly_feature_flags.py
 

.github/workflows/master_fuzzer_binaries.yml

@@ -35,8 +35,9 @@ jobs:
           sudo mkswap $SWAP_FILE
           sudo swapon $SWAP_FILE
 
-      - name: Install cargo fuzz subcommand crate
-        run: cargo install cargo-fuzz
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-bolero
 
       - run: rustup target add --toolchain nightly wasm32-unknown-unknown
 
@@ -47,6 +48,8 @@ jobs:
 
       - uses: actions/checkout@master
 
-      - run: pip install -r scripts/build_fuzzers_requirements.txt
-
-      - run: python3 scripts/build_fuzzers.py ${{ github.ref_name }}
+      - name: "Compile fuzzers and upload to GCS"
+        run: |
+          NAME="nearcore-${{ github.ref_name }}-$(env TZ=Etc/UTC  date +"%Y%m%d%H%M%S")"
+          cargo +nightly bolero build-clusterfuzz --all-features --profile fuzz
+          gsutil cp -Z target/fuzz/clusterfuzz.tar "gs://fuzzer_targets/${{ github.ref_name }}/$NAME.tar.gz"

.github/workflows/ondemand_fuzzer_binaries.yml

@@ -51,8 +51,9 @@ jobs:
           sudo mkswap $SWAP_FILE
           sudo swapon $SWAP_FILE
 
-      - name: Install cargo fuzz subcommand crate
-        run: cargo install cargo-fuzz
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-bolero
 
       - run: rustup target add --toolchain nightly wasm32-unknown-unknown
 
@@ -69,16 +70,20 @@ jobs:
         with:
           ref: ${{ github.event.inputs.branch_ref }}
 
-      - run: pip install -r scripts/build_fuzzers_requirements.txt
-
       - name: Build Release branch fuzz targets
         if: ${{ github.event.action == 'released'}}
-        run: python3 scripts/build_fuzzers.py release
+        run: echo "branch_type=release" >> "$GITHUB_ENV"
 
       - name: Build RC branch fuzz targets
         if: ${{ github.event.action == 'prereleased'}}
-        run: python3 scripts/build_fuzzers.py rc
+        run: echo "branch_type=rc" >> "$GITHUB_ENV"
 
       - name: Build fuzz targets from ${{ github.event.inputs.branch_ref }}" branch
         if: ${{ github.event_name == 'workflow_dispatch'}}
-        run: python3 scripts/build_fuzzers.py ${{ github.event.inputs.branch_type }}
+        run: echo "branch_type=${{ github.event.inputs.branch_type }}" >> "$GITHUB_ENV"
+
+      - name: "Compile fuzzers and upload to GCS"
+      - run: |
+          NAME="nearcore-$branch_type-$(env TZ=Etc/UTC  date +"%Y%m%d%H%M%S")"
+          cargo +nightly bolero build-clusterfuzz --all-features --profile fuzz
+          gsutil cp -Z target/fuzz/clusterfuzz.tar "gs://fuzzer_targets/$branch_type/$NAME.tar.gz"

Cargo.toml

@@ -32,8 +32,6 @@ members = [
     "chain/rosetta-rpc",
     "chain/telemetry",
     "core/async",
-    "core/account-id",
-    "core/account-id/fuzz",
     "core/chain-configs",
     "core/crypto",
     "core/dyn-configs",
@@ -77,6 +75,7 @@ members = [
     "tools/rpctypegen/macro",
     "tools/speedy_sync",
     "tools/state-parts",
+    "tools/state-parts-dump-check",
     "tools/state-viewer",
     "tools/storage-usage-delta-calculator",
     "tools/themis",
@@ -109,8 +108,7 @@ bencher = "0.1.5"
 bitflags = "1.2"
 blake2 = "0.9.1"
 bn = { package = "zeropool-bn", version = "0.5.11" }
-# TODO: revert to upstream bolero once https://github.com/camshaft/bolero/pull/186 lands and gets released
-bolero = { version = "0.9.0", git = "https://github.com/camshaft/bolero", rev = "7d955a67fbd06f61ffaf28178e976280c33c045a", features = ["arbitrary"] }
+bolero = { version = "0.10.0", features = ["arbitrary"] }
 borsh = { version = "1.0.0", features = ["derive", "rc"] }
 bs58 = "0.4"
 bytes = "1"
@@ -159,7 +157,7 @@ hyper-tls = "0.5.0"
 im = "15"
 indexmap = "1.6"
 indicatif = { version = "0.15.0", features = ["with_rayon"] }
-insta = { version = "1.26.0", features = ["json", "yaml"] }
+insta = { version = "1.34.0", features = ["json", "yaml", "redactions"] }
 integration-tests = { path = "integration-tests" }
 itertools = "0.10.0"
 itoa = "1.0"
@@ -174,7 +172,7 @@ lru = "0.7.2"
 memmap2 = "0.5"
 memoffset = "0.8"
 more-asserts = "0.2"
-near-account-id = { path = "core/account-id", features = ["internal_unstable"] }
+near-account-id = { version = "1.0.0-alpha.1", features = ["internal_unstable", "serde", "borsh"] }
 near-actix-test-utils = { path = "test-utils/actix-test-utils" }
 near-amend-genesis = { path = "tools/amend-genesis" }
 near-database-tool = { path = "tools/database" }
@@ -220,6 +218,7 @@ near-rpc-error-core = { path = "tools/rpctypegen/core" }
 near-rpc-error-macro = { path = "tools/rpctypegen/macro" }
 near-stable-hasher = { path = "utils/near-stable-hasher" }
 near-state-parts = { path = "tools/state-parts" }
+near-state-parts-dump-check = { path = "tools/state-parts-dump-check" }
 near-state-viewer = { path = "tools/state-viewer", package = "state-viewer" }
 near-store = { path = "core/store" }
 near-telemetry = { path = "chain/telemetry" }
@@ -397,6 +396,12 @@ inherits = "release"
 lto = false
 codegen-units = 16
 
+# Used for fuzzing, LTO is ill-supported as of 2023-09 and so should not be enabled.
+[profile.fuzz]
+inherits = "dev"
+opt-level = 3
+incremental = false
+codegen-units = 1
 
 # Compile some dependencies with optimizations to speed up tests.
 [profile.dev.package.hex]

chain/chain/Cargo.toml

@@ -57,9 +57,13 @@ protocol_feature_reject_blocks_with_outdated_protocol_version = [
 protocol_feature_simple_nightshade_v2 = [
   "near-primitives/protocol_feature_simple_nightshade_v2",
 ]
+protocol_feature_chunk_validation = [
+  "near-primitives/protocol_feature_chunk_validation",
+]
 
 nightly = [
   "nightly_protocol",
+  "protocol_feature_chunk_validation",
   "protocol_feature_reject_blocks_with_outdated_protocol_version",
   "protocol_feature_simple_nightshade_v2",
   "near-chain-configs/nightly",

chain/chain/src/test_utils/kv_runtime.rs

@@ -524,6 +524,8 @@ impl EpochManagerAdapter for MockEpochManager {
             1,
             1,
             RngSeed::default(),
+            #[cfg(feature = "protocol_feature_chunk_validation")]
+            Default::default(),
         )))
     }
 

chain/client/src/test_utils/test_env.rs

@@ -435,8 +435,10 @@ impl TestEnv {
         relayer: AccountId,
         receiver_id: AccountId,
     ) -> SignedTransaction {
-        let inner_signer = InMemorySigner::from_seed(sender.clone(), KeyType::ED25519, &sender);
-        let relayer_signer = InMemorySigner::from_seed(relayer.clone(), KeyType::ED25519, &relayer);
+        let inner_signer =
+            InMemorySigner::from_seed(sender.clone(), KeyType::ED25519, sender.as_str());
+        let relayer_signer =
+            InMemorySigner::from_seed(relayer.clone(), KeyType::ED25519, relayer.as_str());
         let tip = self.clients[0].chain.head().unwrap();
         let user_nonce = tip.height + 1;
         let relayer_nonce = tip.height + 1;

chain/client/src/tests/bug_repros.rs

@@ -315,7 +315,7 @@ fn test_long_gap_between_blocks() {
                             if approval_message.approval.target_height < target_height {
                                 (NetworkResponses::NoResponse.into(), false)
                             } else {
-                                if approval_message.target.as_ref() == "test1" {
+                                if approval_message.target == "test1" {
                                     (NetworkResponses::NoResponse.into(), true)
                                 } else {
                                     (NetworkResponses::NoResponse.into(), false)

chain/client/src/tests/catching_up.rs

@@ -17,6 +17,7 @@ use near_network::types::{AccountIdOrPeerTrackingShard, PeerInfo};
 use near_network::types::{NetworkRequests, NetworkResponses, PeerManagerMessageRequest};
 use near_o11y::testonly::init_integration_logger;
 use near_o11y::WithSpanContextExt;
+use near_primitives::account::id::AccountIdRef;
 use near_primitives::hash::{hash as hash_func, CryptoHash};
 use near_primitives::network::PeerId;
 use near_primitives::receipt::Receipt;
@@ -689,8 +690,8 @@ fn test_chunk_grieving() {
         let archive = vec![false; vs.all_block_producers().count()];
         let epoch_sync_enabled = vec![true; vs.all_block_producers().count()];
 
-        let malicious_node = "test3.6".parse().unwrap();
-        let victim_node = "test3.5".parse().unwrap();
+        let malicious_node = AccountIdRef::new_or_panic("test3.6");
+        let victim_node = AccountIdRef::new_or_panic("test3.5");
         let phase = Arc::new(RwLock::new(ChunkGrievingPhases::FirstAttack));
         let grieving_chunk_hash = Arc::new(RwLock::new(ChunkHash::default()));
         let unaccepted_block_hash = Arc::new(RwLock::new(CryptoHash::default()));

chain/epoch-manager/Cargo.toml

@@ -33,8 +33,12 @@ expensive_tests = []
 protocol_feature_fix_staking_threshold = [
   "near-primitives/protocol_feature_fix_staking_threshold",
 ]
+protocol_feature_chunk_validation = [
+  "near-primitives/protocol_feature_chunk_validation",
+]
 nightly = [
   "nightly_protocol",
+  "protocol_feature_chunk_validation",
   "protocol_feature_fix_staking_threshold",
   "near-chain-configs/nightly",
   "near-primitives/nightly",