Merge pull request #803 from AleoHQ/parallelize-merkle-tree

Merkle tree parallelize integration + upgrade to snarkVM 0.4.0
ProvableHQ · Jun 1, 2021 · 0774aee · 0774aee
2 parents e811037 + 59f366a
commit 0774aee
Show file tree

Hide file tree

Showing 16 changed files with 283 additions and 155 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -45,20 +45,20 @@ name = "snarkos"
 path = "snarkos/main.rs"
 
 [dependencies.snarkvm-algorithms]
-version = "0.3.1"
+version = "0.4.0"
 default-features = false
 
 [dependencies.snarkvm-dpc]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-objects]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-posw]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-utilities]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkos-consensus]
 path = "./consensus"

diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
@@ -22,13 +22,36 @@ name = "syncing"
 path = "syncing/syncing.rs"
 harness = false
 
+[[bench]]
+name = "merkle"
+path = "merkle/merkle.rs"
+harness = false
+
 [[bench]]
 name = "network"
 path = "network/network.rs"
 harness = false
 
+[dependencies.snarkvm-algorithms]
+version = "0.4.0"
+
+[dependencies.snarkvm-curves]
+version = "0.4.0"
+
+[dependencies.snarkvm-dpc]
+version = "0.4.0"
+
+[dependencies.snarkvm-objects]
+version = "0.4.0"
+
+[dependencies.snarkvm-parameters]
+version = "0.4.0"
+
+[dependencies.snarkvm-posw]
+version = "0.4.0"
+
 [dependencies.snarkvm-utilities]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkos-profiler]
 path = "../profiler"

diff --git a/benchmarks/merkle/merkle.rs b/benchmarks/merkle/merkle.rs
@@ -0,0 +1,54 @@
+// Copyright (C) 2019-2021 Aleo Systems Inc.
+// This file is part of the snarkOS library.
+
+// The snarkOS library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The snarkOS library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the snarkOS library. If not, see <https://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use criterion::*;
+use rand::Rng;
+use snarkvm_algorithms::{merkle_tree::MerkleTree, MerkleParameters, CRH};
+use snarkvm_dpc::base_dpc::instantiated::*;
+use snarkvm_parameters::{LedgerMerkleTreeParameters, Parameter};
+use snarkvm_utilities::bytes::FromBytes;
+
+fn merkle_build(c: &mut Criterion) {
+    let crh_parameters =
+        <MerkleTreeCRH as CRH>::Parameters::read(&LedgerMerkleTreeParameters::load_bytes().unwrap()[..])
+            .expect("read bytes as hash for MerkleParameters in ledger");
+    let merkle_tree_hash_parameters = <CommitmentMerkleParameters as MerkleParameters>::H::from(crh_parameters);
+    let ledger_merkle_tree_parameters: Arc<CommitmentMerkleParameters> =
+        Arc::new(From::from(merkle_tree_hash_parameters));
+
+    let mut commitments = Vec::with_capacity(10000);
+    for _ in 0..10000 {
+        let mut buf = [0u8; 32];
+        rand::thread_rng().fill(&mut buf[..]);
+        commitments.push(buf);
+    }
+
+    c.bench_function("merkle_build", move |b| {
+        b.iter(|| {
+            MerkleTree::new(ledger_merkle_tree_parameters.clone(), &commitments[..]).unwrap();
+        });
+    });
+}
+
+criterion_group! {
+    name = merkle;
+    config = Criterion::default().sample_size(10);
+    targets = merkle_build
+}
+
+criterion_main!(merkle);
diff --git a/consensus/Cargo.toml b/consensus/Cargo.toml
@@ -18,22 +18,22 @@ license = "GPL-3.0"
 edition = "2018"
 
 [dependencies.snarkvm-algorithms]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-curves]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-dpc]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-objects]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-posw]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-utilities]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkos-profiler]
 path = "../profiler"

diff --git a/metrics/Cargo.toml b/metrics/Cargo.toml
@@ -36,7 +36,7 @@ features = [ "macros", "rt-multi-thread" ]
 version = "0.3"
 
 [dev-dependencies.snarkvm-derives]
-version = "0.3.1"
+version = "0.4.0"
 
 [dev-dependencies.serial_test]
 version = "0.5.0"
diff --git a/network/Cargo.toml b/network/Cargo.toml
@@ -18,16 +18,16 @@ license = "GPL-3.0"
 edition = "2018"
 
 [dependencies.snarkvm-algorithms]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-dpc]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-objects]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-utilities]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkos-consensus]
 path = "../consensus"

diff --git a/parameters/Cargo.toml b/parameters/Cargo.toml
@@ -18,34 +18,34 @@ license = "GPL-3.0"
 edition = "2018"
 
 [dependencies.snarkvm-algorithms]
-version = "0.3.1"
+version = "0.4.0"
 default-features = false
 
 [dependencies.snarkvm-parameters]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-utilities]
-version = "0.3.1"
+version = "0.4.0"
 default-features = false
 
 [dependencies.curl]
 version = "0.4.36"
 optional = true
 
 [dev-dependencies.snarkvm-curves]
-version = "0.3.1"
+version = "0.4.0"
 
 [dev-dependencies.snarkvm-dpc]
-version = "0.3.1"
+version = "0.4.0"
 
 [dev-dependencies.snarkvm-marlin]
-version = "0.3.1"
+version = "0.4.0"
 
 [dev-dependencies.snarkvm-objects]
-version = "0.3.1"
+version = "0.4.0"
 
 [dev-dependencies.snarkvm-posw]
-version = "0.3.1"
+version = "0.4.0"
 
 [dev-dependencies.snarkos-consensus]
 path = "../consensus"

diff --git a/parameters/examples/generate_transaction.rs b/parameters/examples/generate_transaction.rs
@@ -57,7 +57,7 @@ fn empty_ledger<T: Transaction, P: LoadableMerkleParameters, S: Storage>(
         .map_err(|err| LedgerError::Message(err.to_string()))?;
 
     let leaves: &[[u8; 32]] = &[];
-    let cm_merkle_tree = MerkleTree::<P>::new(parameters.clone(), leaves.iter())?;
+    let cm_merkle_tree = MerkleTree::<P>::new(parameters.clone(), &leaves[..])?;
 
     Ok(Ledger {
         current_block_height: Default::default(),

diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml
@@ -18,17 +18,17 @@ license = "GPL-3.0"
 edition = "2018"
 
 [dependencies.snarkvm-algorithms]
-version = "0.3.1"
+version = "0.4.0"
 default-features = false
 
 [dependencies.snarkvm-dpc]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-objects]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-utilities]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkos-consensus]
 path = "../consensus"

diff --git a/storage/Cargo.toml b/storage/Cargo.toml
@@ -24,19 +24,19 @@ mem_storage = [ ]
 test = [ ] # enables database deletion for tests; used by snarkos-testing
 
 [dependencies.snarkvm-algorithms]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-dpc]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-objects]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-parameters]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-utilities]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkos-parameters]
 path = "../parameters"
@@ -72,7 +72,7 @@ version = "1.0"
 path = "../consensus"
 
 [dev-dependencies.snarkvm-curves]
-version = "0.3.1"
+version = "0.4.0"
 
 [dev-dependencies.snarkos-testing]
 path = "../testing"
diff --git a/storage/src/ledger.rs b/storage/src/ledger.rs
@@ -134,17 +134,19 @@ impl<T: Transaction, P: LoadableMerkleParameters, S: Storage> Ledger<T, P, S> {
 
                 let mut cm_and_indices = vec![];
 
-                for (commitment_key, index_value) in storage.get_col(COL_COMMITMENT)? {
+                let cms = storage.get_col(COL_COMMITMENT)?;
+
+                for (commitment_key, index_value) in cms {
                     let commitment: T::Commitment = FromBytes::read(&commitment_key[..])?;
                     let index = bytes_to_u32(&index_value) as usize;
 
                     cm_and_indices.push((commitment, index));
                 }
 
                 cm_and_indices.sort_by(|&(_, i), &(_, j)| i.cmp(&j));
-                let commitments = cm_and_indices.into_iter().map(|(cm, _)| cm);
+                let commitments = cm_and_indices.into_iter().map(|(cm, _)| cm).collect::<Vec<_>>();
 
-                let merkle_tree = MerkleTree::new(ledger_parameters.clone(), commitments)?;
+                let merkle_tree = MerkleTree::new(ledger_parameters.clone(), &commitments[..])?;
 
                 Ok(Self {
                     current_block_height: AtomicU32::new(bytes_to_u32(&val)),

diff --git a/storage/src/objects/dpc_state.rs b/storage/src/objects/dpc_state.rs
@@ -120,9 +120,13 @@ impl<T: Transaction, P: LoadableMerkleParameters, S: Storage> Ledger<T, P, S> {
         new_cm_and_indices.sort_by(|&(_, i), &(_, j)| i.cmp(&j));
 
         let old_commitments = old_cm_and_indices.into_iter().map(|(cm, _)| cm);
-        let new_commitments = new_cm_and_indices.into_iter().map(|(cm, _)| cm);
+        let new_commitments = new_cm_and_indices.into_iter().map(|(cm, _)| cm).collect::<Vec<_>>();
 
-        let new_tree = { self.cm_merkle_tree.read().rebuild(old_commitments, new_commitments)? };
+        let new_tree = {
+            self.cm_merkle_tree
+                .read()
+                .rebuild(old_commitments, &new_commitments[..])?
+        };
         *self.cm_merkle_tree.write() = new_tree;
 
         Ok(())

diff --git a/storage/src/objects/ledger_scheme.rs b/storage/src/objects/ledger_scheme.rs
@@ -56,7 +56,7 @@ impl<T: Transaction, P: LoadableMerkleParameters, S: Storage> LedgerScheme for L
         }
 
         let leaves: &[[u8; 32]] = &[];
-        let empty_cm_merkle_tree = MerkleTree::<Self::MerkleParameters>::new(parameters.clone(), leaves.iter())?;
+        let empty_cm_merkle_tree = MerkleTree::<Self::MerkleParameters>::new(parameters.clone(), leaves)?;
 
         let ledger_storage = Self {
             current_block_height: Default::default(),

diff --git a/testing/Cargo.toml b/testing/Cargo.toml
@@ -29,25 +29,25 @@ network = [
 ]
 
 [dependencies.snarkvm-algorithms]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-curves]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-dpc]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-objects]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-parameters]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-posw]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkvm-utilities]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.snarkos-consensus]
 path = "../consensus"

diff --git a/toolkit/Cargo.toml b/toolkit/Cargo.toml
@@ -26,20 +26,20 @@ path = "benches/account.rs"
 harness = false
 
 [dependencies.snarkvm-algorithms]
-version = "0.3.1"
+version = "0.4.0"
 default-features = false
 features = [ "wasm" ]
 
 [dependencies.snarkvm-dpc]
-version = "0.3.1"
+version = "0.4.0"
 default-features = false
 
 [dependencies.snarkvm-objects]
-version = "0.3.1"
+version = "0.4.0"
 default-features = false
 
 [dependencies.snarkvm-utilities]
-version = "0.3.1"
+version = "0.4.0"
 
 [dependencies.anyhow]
 version = "1.0.40"