From 9ef7ef93241891c7cd2fc78d3af8119b0ab10279 Mon Sep 17 00:00:00 2001 From: Shuwen Sun Date: Sat, 26 Oct 2024 20:56:39 +0000 Subject: [PATCH] load and dump map --- meson.build | 8 ++ src/include/zstore_controller.h | 23 +++-- src/main.cc | 47 ++++++---- src/tests/map_test.cc | 160 ++++++++++++++++++++++++++++++++ src/zstore_controller.cc | 158 ++++++++++++++++++++++++++++++- 5 files changed, 365 insertions(+), 31 deletions(-) create mode 100644 src/tests/map_test.cc diff --git a/meson.build b/meson.build index 0d3e38d..f7bcd6a 100644 --- a/meson.build +++ b/meson.build @@ -303,6 +303,14 @@ executable( # install: true, # ) +executable( + 'test_map', + files('src/tests/map_test.cc'), + dependencies: [zstore_deps] + [dependency('_spdk')], + link_args: spdk_link_args, + install: true, +) + executable( 'test_simple_object', files('src/tests/simple_object_test.cc'), diff --git a/src/include/zstore_controller.h b/src/include/zstore_controller.h index d548054..cb23496 100644 --- a/src/include/zstore_controller.h +++ b/src/include/zstore_controller.h @@ -12,8 +12,8 @@ #include namespace net = boost::asio; // from -using zstore_map = boost::concurrent_flat_map; -using zstore_bloom_filter = boost::concurrent_flat_set; +using ZstoreMap = boost::concurrent_flat_map; +using ZstoreBloomFilter = boost::concurrent_flat_set; class Device; class Zone; @@ -22,7 +22,13 @@ class ZstoreController { public: int PopulateDevHash(); - int PopulateMap(bool bogus); + int PopulateMap(); + Result DumpAllMap(); + Result ReadAllMap(); + + void writeMapToFile(const std::string &filename); + void readMapFromFile(const std::string &filename); + // Result PopulateMap(bool bogus, int key_experiment); // Result PopulateDevHash(int key_experiment); @@ -38,14 +44,14 @@ class ZstoreController std::shared_mutex mDevHashMutex; // ZStore Map: this maps key to tuple of ZNS target and lba - zstore_map mMap; + ZstoreMap mMap; // ZStore Bloom Filter: this maintains a bloom filter of hashes of // object name (key). // // For simplicity, right now we are just using a set to keep track of // the hashes - zstore_bloom_filter mBF; + ZstoreBloomFilter mBF; // Object APIs Result SearchBF(const ObjectKeyHash &key_hash); @@ -64,7 +70,7 @@ class ZstoreController net::io_context &mIoc_; ~ZstoreController(); - int Init(bool object, int key_experiment); + int Init(bool object, int key_experiment, int phase); // threads void initIoThread(); @@ -83,6 +89,7 @@ class ZstoreController int GetQueueDepth() { return mQueueDepth; }; void setQueuDepth(int queue_depth) { mQueueDepth = queue_depth; }; void setKeyExperiment(int key) { mKeyExperiment = key; }; + void setPhase(int phase) { mPhase = phase; }; void SetEventPoller(spdk_poller *p) { mEventsPoller = p; } void SetCompletionPoller(spdk_poller *p) { mCompletionPoller = p; } @@ -218,6 +225,10 @@ class ZstoreController // 6: GC // 7: Checkpoint + int mPhase; + // 1: prepare + // 2: run + private: // number of devices int mN; diff --git a/src/main.cc b/src/main.cc index 1002921..5bd941a 100644 --- a/src/main.cc +++ b/src/main.cc @@ -3,6 +3,7 @@ #include "include/zstore_controller.h" #include "spdk/env.h" #include "spdk/thread.h" +#include "src/include/utils.h" #include #include #include @@ -21,7 +22,7 @@ int main(int argc, char **argv) } int key_experiment = std::stoull(argv[1]); - int dummy = std::stoull(argv[2]); + int phase = std::stoull(argv[2]); int rc; struct spdk_env_opts opts; @@ -53,30 +54,36 @@ int main(int argc, char **argv) net::io_context ioc{Configuration::GetNumHttpThreads()}; gZstoreController = new ZstoreController(std::ref(ioc)); - gZstoreController->Init(false, key_experiment); + gZstoreController->Init(false, key_experiment, phase); log_info("Starting HTTP server with port 2000!\n"); // gZstoreController->mIoc_.run(); // FIXME only tput on Zstore2Dev1 - while (1) { - // auto etime = std::chrono::high_resolution_clock::now(); - // auto delta = - // std::chrono::duration_cast( - // etime - gZstoreController->stime) - // .count(); - // auto tput = - // gZstoreController->mTotalCounts * g_micro_to_second / - // delta; - // - // log_info("Total IO {}, total time {}ms, throughput {} IOPS", - // gZstoreController->mTotalCounts, delta, tput); - // { - // gZstoreController->stime = - // std::chrono::high_resolution_clock::now(); - // gZstoreController->mTotalCounts = 0; - // } - sleep(1); + // while (1) { + // auto etime = std::chrono::high_resolution_clock::now(); + // auto delta = + // std::chrono::duration_cast( + // etime - gZstoreController->stime) + // .count(); + // auto tput = + // gZstoreController->mTotalCounts * g_micro_to_second / + // delta; + // + // log_info("Total IO {}, total time {}ms, throughput {} IOPS", + // gZstoreController->mTotalCounts, delta, tput); + // { + // gZstoreController->stime = + // std::chrono::high_resolution_clock::now(); + // gZstoreController->mTotalCounts = 0; + // } + // sleep(1); + // } + + sleep(30); + if (gZstoreController->mPhase == 2 && gZstoreController->mPhase == 1) { + log_info("Prepare phase is done, dumping all map and bloom filter"); + gZstoreController->DumpAllMap(); } gZstoreController->Drain(); diff --git a/src/tests/map_test.cc b/src/tests/map_test.cc new file mode 100644 index 0000000..51c9e64 --- /dev/null +++ b/src/tests/map_test.cc @@ -0,0 +1,160 @@ +#include "../include/types.h" +#include "../include/zstore_controller.h" +#include +#include +#include +#include + +using namespace std; + +// Type definitions +using ObjectKey = std::string; +using ObjectKeyHash = unsigned char *; +using TargetDev = std::string; +using Lba = uint64_t; +using Length = uint32_t; + +// Device types and map entry +using TargetLbaTuple = std::tuple; +using MapEntry = std::tuple; +// using MapType = boost::container::concurrent_flat_map; + +// Helper function to write a single string to the file +void writeString(std::ofstream &outFile, const std::string &str) +{ + size_t size = str.size(); + outFile.write(reinterpret_cast(&size), sizeof(size)); + outFile.write(str.c_str(), size); +} + +// Helper function to read a single string from the file +std::string readString(std::ifstream &inFile) +{ + size_t size; + inFile.read(reinterpret_cast(&size), sizeof(size)); + std::string str(size, '\0'); + inFile.read(&str[0], size); + return str; +} + +// Function to write the map to a file +void writeMapToFile(const ZstoreMap &map, const std::string &filename) +{ + std::ofstream outFile(filename, std::ios::binary); + if (!outFile) { + std::cerr << "Error opening file for writing: " << filename + << std::endl; + return; + } + + size_t mapSize = 0; + map.visit_all([&mapSize](auto &x) { ++mapSize; }); + + // Write the map size + outFile.write(reinterpret_cast(&mapSize), sizeof(mapSize)); + + // Iterate over each key-value pair and write them to the file + map.visit_all([&outFile](auto &x) { + // Serialize the key + outFile.write(reinterpret_cast(x.first), + 32); // Assuming 32-byte hash + + // Serialize the entry + auto writeTargetLbaTuple = [&outFile](const TargetLbaTuple &tuple) { + writeString(outFile, std::get<0>(tuple)); // TargetDev + outFile.write(reinterpret_cast(&std::get<1>(tuple)), + sizeof(Lba)); // Lba + outFile.write(reinterpret_cast(&std::get<2>(tuple)), + sizeof(Length)); // Length + }; + writeTargetLbaTuple(std::get<0>(x.second)); + writeTargetLbaTuple(std::get<1>(x.second)); + writeTargetLbaTuple(std::get<2>(x.second)); + }); +} + +// Function to read the map from a file +ZstoreMap readMapFromFile(const std::string &filename) +{ + std::ifstream inFile(filename, std::ios::binary); + if (!inFile) { + std::cerr << "Error opening file for reading: " << filename + << std::endl; + return {}; + } + + ZstoreMap map; + size_t mapSize; + inFile.read(reinterpret_cast(&mapSize), sizeof(mapSize)); + + for (size_t i = 0; i < mapSize; ++i) { + // Deserialize the key + unsigned char *key = new unsigned char[32]; + inFile.read(reinterpret_cast(key), 32); + + // Deserialize the entry + auto readTargetLbaTuple = [&inFile]() -> TargetLbaTuple { + std::string dev = readString(inFile); + Lba lba; + Length length; + inFile.read(reinterpret_cast(&lba), sizeof(Lba)); + inFile.read(reinterpret_cast(&length), sizeof(Length)); + return std::make_tuple(dev, lba, length); + }; + + MapEntry entry = std::make_tuple( + readTargetLbaTuple(), readTargetLbaTuple(), readTargetLbaTuple()); + map.emplace(key, entry); + } + + return map; +} + +int main() +{ + ZstoreMap mMap; + + // Sample data + unsigned char key1[32] = "sample_key_1"; + unsigned char key2[32] = "sample_key_2"; + MapEntry entry1 = {std::make_tuple("device1", 1000, 256), + std::make_tuple("device2", 2000, 512), + std::make_tuple("device3", 3000, 128)}; + MapEntry entry2 = {std::make_tuple("device1", 1500, 128), + std::make_tuple("device2", 2500, 256), + std::make_tuple("device3", 3500, 64)}; + mMap.emplace(key1, entry1); + mMap.emplace(key2, entry2); + + // Write the map to a file + std::string filename = "mMap_data.bin"; + writeMapToFile(mMap, filename); + + // Read the map back from the file + ZstoreMap loadedMap = readMapFromFile(filename); + + // Print the loaded map + loadedMap.visit_all([](auto &x) { + std::cout << "Key: " << x.first << "\n"; + std::tuple< + std::tuple, unsigned long, unsigned int>, + std::tuple, unsigned long, unsigned int>, + std::tuple, unsigned long, unsigned int>> + big_tuple = x.second; + auto first = std::get<0>(big_tuple); + std::cout << " Device: " << std::get<0>(first) + << ", LBA: " << std::get<1>(first) + << ", Length: " << std::get<2>(first) << "\n"; + auto second = std::get<1>(big_tuple); + std::cout << " Device: " << std::get<0>(second) + << ", LBA: " << std::get<1>(second) + << ", Length: " << std::get<2>(second) << "\n"; + auto third = std::get<2>(big_tuple); + std::cout << " Device: " << std::get<0>(third) + << ", LBA: " << std::get<1>(third) + << ", Length: " << std::get<2>(third) << "\n"; + }); + + return 0; +} diff --git a/src/zstore_controller.cc b/src/zstore_controller.cc index 5fbfab8..7f854e9 100644 --- a/src/zstore_controller.cc +++ b/src/zstore_controller.cc @@ -5,8 +5,11 @@ #include "include/global.h" #include "include/http_server.h" #include "include/object.h" +#include +#include #include #include +#include #include int zone_offset = 1808277; @@ -186,10 +189,11 @@ void ZstoreController::unregister_controllers(std::vector &g_devices) // struct spdk_nvme_detach_ctx *detach_ctx = NULL; } -int ZstoreController::PopulateMap(bool bogus) +int ZstoreController::PopulateMap() { + log_info("populate map: mKeyExperiment {}, mPhase {}", mKeyExperiment, + mPhase); unsigned char hash[SHA256_DIGEST_LENGTH]; - // static unsigned char buffer[65]; if (mKeyExperiment == 1) { // Random Read for (int i = 0; i < 2'000'000; i++) { @@ -204,11 +208,34 @@ int ZstoreController::PopulateMap(bool bogus) i + zone_offset, 1, i + zone_offset, 1, i + zone_offset, 1) .value(); - // assert(rc.has_value()); sha256("/db/" + std::to_string(i), hash); mMap.insert({hash, entry}); } } else if (mKeyExperiment == 2) { + if (mPhase == 1) { + log_info("Prepare phase, do nothing"); + for (int i = 0; i < 2'000; i++) { + auto entry = + createMapEntry( + std::make_tuple( + std::make_pair("Zstore2Dev1", + Configuration::GetZoneId1()), + std::make_pair("Zstore2Dev2", + Configuration::GetZoneId1()), + std::make_pair("Zstore3Dev1", + Configuration::GetZoneId2())), + i + zone_offset, 1, i + zone_offset, 1, i + zone_offset, + 1) + .value(); + sha256("/db/" + std::to_string(i), hash); + mMap.insert({hash, entry}); + } + DumpAllMap(); + } else if (mPhase == 2) { + log_info("Run phase, load the map and the bloom filter"); + ReadAllMap(); + } + // Sequential write (append) and read // for (int i = 0; i < 2'000'000; i++) { // mMap.insert({"/db/" + std::to_string(i), @@ -229,6 +256,126 @@ int ZstoreController::PopulateMap(bool bogus) return 0; } +// Helper function to write a string to a binary file +void writeString(std::ofstream &outFile, const std::string &str) +{ + size_t length = str.size(); + outFile.write(reinterpret_cast(&length), sizeof(length)); + outFile.write(str.data(), length); +} + +// Helper function to read a string from a binary file +std::string readString(std::ifstream &inFile) +{ + size_t length; + inFile.read(reinterpret_cast(&length), sizeof(length)); + std::string str(length, '\0'); + inFile.read(&str[0], length); + return str; +} + +// Function to write the map to a file +void ZstoreController::writeMapToFile(const std::string &filename) +{ + log_debug("write map to file : {}", filename); + std::ofstream outFile(filename, std::ios::binary); + if (!outFile) { + std::cerr << "Error opening file for writing: " << filename + << std::endl; + return; + } + + log_debug("1111"); + // Write the size of the map + size_t mapSize = mMap.size(); + outFile.write(reinterpret_cast(&mapSize), sizeof(mapSize)); + + log_debug("2222"); + mMap.visit_all([&](auto &kv) { + log_debug("kv "); + // Write each key-value pair + // for (const auto &[key, value] : map) { + // Serialize the key (fixed-length hash) + outFile.write(reinterpret_cast(kv.first), kHashSize); + + log_debug("333"); + // Serialize the value (MapEntry) + for (const auto &targetTuple : + {std::get<0>(kv.second), std::get<1>(kv.second), + std::get<2>(kv.second)}) { + log_debug("123"); + // Write TargetDev + // writeString(outFile, std::get<0>(targetTuple)); + + log_debug("444"); + // Write Lba and Length + outFile.write( + reinterpret_cast(&std::get<1>(targetTuple)), + sizeof(Lba)); + log_debug("444"); + outFile.write( + reinterpret_cast(&std::get<2>(targetTuple)), + sizeof(Length)); + } + }); +} + +// Function to read the map from a file +void ZstoreController::readMapFromFile(const std::string &filename) +{ + log_debug("read map from file : {}", filename); + std::ifstream inFile(filename, std::ios::binary); + if (!inFile) { + std::cerr << "Error opening file for reading: " << filename + << std::endl; + } + + // Read the size of the map + size_t mapSize; + inFile.read(reinterpret_cast(&mapSize), sizeof(mapSize)); + + // Read each key-value pair + for (size_t i = 0; i < mapSize; ++i) { + // Deserialize the key (fixed-length hash) + unsigned char key[kHashSize]; + inFile.read(reinterpret_cast(key), kHashSize); + + // Deserialize the value (MapEntry) + TargetLbaTuple t1, t2, t3; + for (auto &targetTuple : {std::ref(t1), std::ref(t2), std::ref(t3)}) { + // Read TargetDev + std::get<0>(targetTuple.get()) = readString(inFile); + + // Read Lba and Length + inFile.read( + reinterpret_cast(&std::get<1>(targetTuple.get())), + sizeof(Lba)); + inFile.read( + reinterpret_cast(&std::get<2>(targetTuple.get())), + sizeof(Length)); + } + + // Insert the key-value pair into the map + mMap.emplace(key, std::make_tuple(t1, t2, t3)); + } +} + +Result ZstoreController::DumpAllMap() +{ + log_debug("dump map "); + // Write the map to a file + std::string filename = "map_data.bin"; + writeMapToFile(filename); +} + +Result ZstoreController::ReadAllMap() +{ + log_debug("read dump map "); + std::string filename = "map_data.bin"; + // Read the map back from the file + readMapFromFile(filename); +} + // this yields a list of devices for a given object key // // right now we create 120 (6*5*4) tuples, to consider ordering of every tuple @@ -282,7 +429,7 @@ int ZstoreController::PopulateDevHash() return 0; } -int ZstoreController::Init(bool object, int key_experiment) +int ZstoreController::Init(bool object, int key_experiment, int phase) { int rc = 0; verbose = Configuration::Verbose(); @@ -292,6 +439,7 @@ int ZstoreController::Init(bool object, int key_experiment) setNumOfDevices(Configuration::GetNumOfDevices() * Configuration::GetNumOfTargets()); setKeyExperiment(key_experiment); + setPhase(phase); // TODO: set all parameters too log_debug("Configuration: sector size {}, queue size {}, context pool size " @@ -417,7 +565,7 @@ int ZstoreController::Init(bool object, int key_experiment) rc = PopulateDevHash(); assert(rc == 0); - rc = PopulateMap(true); + rc = PopulateMap(); assert(rc == 0); pivot = 0;