From f48dc46ac6ad757e733a778abea319e6c896c749 Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Wed, 3 May 2023 15:57:15 +0300 Subject: [PATCH 1/2] Added test with multiple MPTs in the same storage space. In this test we create 201 addresses and for each address we create a tree and fill it with random data. In the end of the test we check that all the extracted paths for each key/value pair correctly reaches the associated Merkle root for the tree. --- kvbc/test/CMakeLists.txt | 9 ++ .../sparse_merkle/multiple_trees_test.cpp | 104 ++++++++++++++++++ kvbc/test/sparse_merkle/test_db.h | 4 + 3 files changed, 117 insertions(+) create mode 100644 kvbc/test/sparse_merkle/multiple_trees_test.cpp diff --git a/kvbc/test/CMakeLists.txt b/kvbc/test/CMakeLists.txt index 1ba740a7a1..a816e55639 100644 --- a/kvbc/test/CMakeLists.txt +++ b/kvbc/test/CMakeLists.txt @@ -112,6 +112,15 @@ target_link_libraries(sparse_merkle_tree_test PUBLIC concord-crypto ) +add_executable(sparse_merkle_multiple_trees_test sparse_merkle/multiple_trees_test.cpp) +target_link_libraries(sparse_merkle_multiple_trees_test PUBLIC + GTest::Main + GTest::GTest + kvbc + corebft + concord-crypto +) + if (BUILD_ROCKSDB_STORAGE) add_executable(multiIO_test multiIO_test.cpp ) add_test(multiIO_test multiIO_test) diff --git a/kvbc/test/sparse_merkle/multiple_trees_test.cpp b/kvbc/test/sparse_merkle/multiple_trees_test.cpp new file mode 100644 index 0000000000..8d0e36dca3 --- /dev/null +++ b/kvbc/test/sparse_merkle/multiple_trees_test.cpp @@ -0,0 +1,104 @@ +// Concord +// +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the sub-component's license, as noted in the +// LICENSE file. + +#include +#include + +#include "gtest/gtest.h" +#include "sparse_merkle/tree.h" +#include "sparse_merkle/proof_path_processor.h" + +#include "test_db.h" + +#include +using namespace std; + +using namespace concordUtils; +using namespace concord::kvbc; +using namespace concord::kvbc::sparse_merkle; +using namespace proof_path_processor; + +void db_put(const shared_ptr& db, const UpdateBatch& batch) { + for (const auto& [key, val] : batch.internal_nodes) { + db->put(key, val); + } + for (const auto& [key, val] : batch.leaf_nodes) { + db->put(key, val); + } + // remove the stale keys/values (we only need the latest for this test) + for (auto& internal_key : batch.stale.internal_keys) { + db->del(internal_key); + } + for (auto& leaf_key : batch.stale.leaf_keys) { + db->del(leaf_key); + } +} + +std::string random_string(size_t length) { + auto randchar = []() -> char { + const char charset[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + const size_t max_index = (sizeof(charset) - 1); + return charset[rand() % max_index]; + }; + std::string str(length, 0); + std::generate_n(str.begin(), length, randchar); + return str; +} + +TEST(tree_tests, fill_trees_then_extract_proof_paths_and_verify_them) { + auto seed = time(0); + srand(seed); + cout << "Test seed: " << seed << endl; + std::shared_ptr db(new TestDB); + std::vector addresses; + addresses.push_back(""); + for (int i = 0; i < 200; i++) { + addresses.push_back("MY_ETH_ADDR_" + std::to_string(i)); + } + + std::vector> all; + all.resize(addresses.size()); + + for (int i = 1; i <= 1500; i++) { + for (size_t address = 0; address < addresses.size(); address++) { + Tree tree(db, addresses[address]); + SetOfKeyValuePairs updates; + for (int j = 1; j <= 10; j++) { + auto key = random_string(256); + auto val = random_string(256); + all[address][key] = val; + updates.insert({Sliver(std::move(key)), Sliver(std::move(val))}); + } + auto batch = tree.update(updates); + db_put(db, batch); + } + } + for (size_t address = 0; address < addresses.size(); address++) { + Tree tree(db, addresses[address]); + auto rootHash = tree.get_root_hash(); + for (const auto& v : all[address]) { + Sliver key{Sliver(std::string(v.first))}; + ASSERT_TRUE( + verifyProofPath(key, Sliver(std::string(v.second)), getProofPath(key, db, addresses[address]), rootHash)); + } + } +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + + int res = RUN_ALL_TESTS(); + return res; +} diff --git a/kvbc/test/sparse_merkle/test_db.h b/kvbc/test/sparse_merkle/test_db.h index 69ed0946e5..f1251eae8b 100644 --- a/kvbc/test/sparse_merkle/test_db.h +++ b/kvbc/test/sparse_merkle/test_db.h @@ -32,6 +32,10 @@ class TestDB : public IDBReader { } } + bool del(const LeafKey& key) { return leaf_nodes_.erase(key) == 1; } + + bool del(const InternalNodeKey& key) { return internal_nodes_.erase(key) == 1; } + BatchedInternalNode get_latest_root(std::string custom_prefix = "") const override { if (latest_version_.find(custom_prefix) == latest_version_.end()) { return BatchedInternalNode(); From 72cbbeb38ef415d3c1928c342bc0ba38b974ea53 Mon Sep 17 00:00:00 2001 From: Hristo Staykov Date: Fri, 5 May 2023 11:51:06 +0300 Subject: [PATCH 2/2] Introduce a test database that stores the keys/values serialized. --- .../sparse_merkle/multiple_trees_test.cpp | 6 +- kvbc/test/sparse_merkle/serialized_test_db.h | 79 +++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 kvbc/test/sparse_merkle/serialized_test_db.h diff --git a/kvbc/test/sparse_merkle/multiple_trees_test.cpp b/kvbc/test/sparse_merkle/multiple_trees_test.cpp index 8d0e36dca3..845b766abb 100644 --- a/kvbc/test/sparse_merkle/multiple_trees_test.cpp +++ b/kvbc/test/sparse_merkle/multiple_trees_test.cpp @@ -17,7 +17,7 @@ #include "sparse_merkle/tree.h" #include "sparse_merkle/proof_path_processor.h" -#include "test_db.h" +#include "serialized_test_db.h" #include using namespace std; @@ -27,7 +27,7 @@ using namespace concord::kvbc; using namespace concord::kvbc::sparse_merkle; using namespace proof_path_processor; -void db_put(const shared_ptr& db, const UpdateBatch& batch) { +void db_put(const shared_ptr& db, const UpdateBatch& batch) { for (const auto& [key, val] : batch.internal_nodes) { db->put(key, val); } @@ -61,7 +61,7 @@ TEST(tree_tests, fill_trees_then_extract_proof_paths_and_verify_them) { auto seed = time(0); srand(seed); cout << "Test seed: " << seed << endl; - std::shared_ptr db(new TestDB); + std::shared_ptr db(new SerializedTestDB); std::vector addresses; addresses.push_back(""); for (int i = 0; i < 200; i++) { diff --git a/kvbc/test/sparse_merkle/serialized_test_db.h b/kvbc/test/sparse_merkle/serialized_test_db.h new file mode 100644 index 0000000000..beabb61ac8 --- /dev/null +++ b/kvbc/test/sparse_merkle/serialized_test_db.h @@ -0,0 +1,79 @@ +// Concord +// +// Copyright (c) 2019 VMware, Inc. All Rights Reserved. +// +// This product is licensed to you under the Apache 2.0 license (the "License"). +// You may not use this product except in compliance with the Apache 2.0 License. +// +// This product may include a number of subcomponents with separate copyright +// notices and license terms. Your use of these subcomponents is subject to the +// terms and conditions of the sub-component's license, as noted in the +// LICENSE file. + +#include + +#include "sparse_merkle/keys.h" +#include "sparse_merkle/db_reader.h" +#include "sparse_merkle/internal_node.h" +#include "kvbc/include/categorization/details.h" +#include "kvbc/include/merkle_tree_serialization.h" + +using namespace std; +using namespace concord::kvbc::sparse_merkle; + +class SerializedTestDB : public IDBReader { + public: + void put(const LeafKey& key, const LeafNode& val) { + std::string serializedLeafKey = concord::kvbc::v2MerkleTree::detail::serialize(key); + std::string serializedLeafNode(val.value.data(), val.value.size()); + ConcordAssert(nodes_.find(serializedLeafKey) == nodes_.end()); + nodes_[serializedLeafKey] = serializedLeafNode; + // std::cout << "DEBUG: " << serializedLeafKey << ":" << serializedLeafNode << "\n"; + } + void put(const InternalNodeKey& key, const BatchedInternalNode& val) { + std::string serializedInternalNodeKey = concord::kvbc::v2MerkleTree::detail::serialize(key); + std::string serializedBatchedInternalNode = concord::kvbc::v2MerkleTree::detail::serialize(val); + + ConcordAssert(nodes_.find(serializedInternalNodeKey) == nodes_.end()); + nodes_[serializedInternalNodeKey] = serializedBatchedInternalNode; + // std::cout << "DEBUG: " << serializedInternalNodeKey << ":" << serializedBatchedInternalNode << "\n"; + + if (latest_version_[key.customPrefix().value()] < key.version()) { + latest_version_[key.customPrefix().value()] = key.version(); + } + } + + bool del(const LeafKey& key) { + std::string serializedLeafKey = concord::kvbc::v2MerkleTree::detail::serialize(key); + return nodes_.erase(serializedLeafKey) == 1; + } + + bool del(const InternalNodeKey& key) { + std::string serializedInternalNodeKey = concord::kvbc::v2MerkleTree::detail::serialize(key); + return nodes_.erase(serializedInternalNodeKey) == 1; + } + + BatchedInternalNode get_latest_root(std::string custom_prefix = "") const override { + if (latest_version_.find(custom_prefix) == latest_version_.end()) { + return BatchedInternalNode(); + } + auto root_key = InternalNodeKey::root(custom_prefix, latest_version_.at(custom_prefix)); + auto ser_root_key = concord::kvbc::v2MerkleTree::detail::serialize(root_key); + ConcordAssert(nodes_.find(ser_root_key) != nodes_.end()); + auto retVal = nodes_.at(ser_root_key); + return concord::kvbc::v2MerkleTree::detail::deserialize( + concordUtils::Sliver{std::move(retVal)}); + } + + BatchedInternalNode get_internal(const InternalNodeKey& key) const override { + auto ser_key = concord::kvbc::v2MerkleTree::detail::serialize(key); + ConcordAssert(nodes_.find(ser_key) != nodes_.end()); + auto retVal = nodes_.at(ser_key); + return concord::kvbc::v2MerkleTree::detail::deserialize( + concordUtils::Sliver{std::move(retVal)}); + } + + private: + map latest_version_; + map nodes_; +};