feat(tool): forest-tool api generate-test-snapshot

src/chain/store/chain_store.rs

@@ -234,7 +234,7 @@ where
                 &self
                     .settings
                     .require_obj::<TipsetKey>(HEAD_KEY)
-                    .expect("failed to load heaviest tipset"),
+                    .expect("failed to load heaviest tipset key"),
             )
             .expect("failed to load heaviest tipset")
     }

src/chain/store/index.rs

@@ -7,6 +7,7 @@ use crate::beacon::{BeaconEntry, IGNORE_DRAND_VAR};
 use crate::blocks::{Tipset, TipsetKey};
 use crate::metrics;
 use crate::shim::clock::ChainEpoch;
+use crate::utils::misc::env::is_env_truthy;
 use fvm_ipld_blockstore::Blockstore;
 use itertools::Itertools;
 use lru::LruCache;
@@ -47,11 +48,13 @@ impl<DB: Blockstore> ChainIndex<DB> {
     /// Loads a tipset from memory given the tipset keys and cache. Semantically
     /// identical to [`Tipset::load`] but the result is cached.
     pub fn load_tipset(&self, tsk: &TipsetKey) -> Result<Option<Arc<Tipset>>, Error> {
-        if let Some(ts) = self.ts_cache.lock().get(tsk) {
-            metrics::LRU_CACHE_HIT
-                .get_or_create(&metrics::values::TIPSET)
-                .inc();
-            return Ok(Some(ts.clone()));
+        if !is_env_truthy("FOREST_TIPSET_CACHE_DISABLED") {
+            if let Some(ts) = self.ts_cache.lock().get(tsk) {
+                metrics::LRU_CACHE_HIT
+                    .get_or_create(&metrics::values::TIPSET)
+                    .inc();
+                return Ok(Some(ts.clone()));
+            }
         }
 
         let ts_opt = Tipset::load(&self.db, tsk)?.map(Arc::new);

src/cli_shared/snapshot.rs

@@ -134,6 +134,9 @@ fn parse_content_disposition(value: &reqwest::header::HeaderValue) -> Option<Str
 
 /// Download the file at `url` with a private HTTP client, returning the path to the downloaded file
 async fn download_http(url: &Url, directory: &Path, filename: &str) -> anyhow::Result<PathBuf> {
+    if !directory.is_dir() {
+        std::fs::create_dir_all(directory)?;
+    }
     let dst_path = directory.join(filename);
     let destination = dst_path.display();
     event!(target: "forest::snapshot", tracing::Level::INFO, %url, %destination, "downloading snapshot");

src/db/memory.rs

@@ -1,6 +1,7 @@
 // Copyright 2019-2024 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
+use super::{EthMappingsStore, SettingsStore};
 use crate::cid_collections::CidHashSet;
 use crate::db::{GarbageCollectable, PersistentStore};
 use crate::libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite};
@@ -11,23 +12,48 @@ use cid::Cid;
 use fvm_ipld_blockstore::Blockstore;
 use itertools::Itertools;
 use parking_lot::RwLock;
-
-use super::{EthMappingsStore, SettingsStore};
+use std::ops::Deref;
 
 #[derive(Debug, Default)]
 pub struct MemoryDB {
-    blockchain_db: RwLock<HashMap<Vec<u8>, Vec<u8>>>,
-    blockchain_persistent_db: RwLock<HashMap<Vec<u8>, Vec<u8>>>,
+    blockchain_db: RwLock<HashMap<Cid, Vec<u8>>>,
+    blockchain_persistent_db: RwLock<HashMap<Cid, Vec<u8>>>,
     settings_db: RwLock<HashMap<String, Vec<u8>>>,
     eth_mappings_db: RwLock<HashMap<EthHash, Vec<u8>>>,
 }
 
+impl MemoryDB {
+    pub fn serialize(&self) -> anyhow::Result<Vec<u8>> {
+        let blockchain_db = self.blockchain_db.read();
+        let blockchain_persistent_db = self.blockchain_persistent_db.read();
+        let settings_db = self.settings_db.read();
+        let eth_mappings_db = self.eth_mappings_db.read();
+        let tuple = (
+            blockchain_db.deref(),
+            blockchain_persistent_db.deref(),
+            settings_db.deref(),
+            eth_mappings_db.deref(),
+        );
+        Ok(fvm_ipld_encoding::to_vec(&tuple)?)
+    }
+
+    pub fn deserialize_from(bytes: &[u8]) -> anyhow::Result<Self> {
+        let (blockchain_db, blockchain_persistent_db, settings_db, eth_mappings_db) =
+            fvm_ipld_encoding::from_slice(bytes)?;
+        Ok(Self {
+            blockchain_db: RwLock::new(blockchain_db),
+            blockchain_persistent_db: RwLock::new(blockchain_persistent_db),
+            settings_db: RwLock::new(settings_db),
+            eth_mappings_db: RwLock::new(eth_mappings_db),
+        })
+    }
+}
+
 impl GarbageCollectable<CidHashSet> for MemoryDB {
     fn get_keys(&self) -> anyhow::Result<CidHashSet> {
         let mut set = CidHashSet::new();
-        for key in self.blockchain_db.read().keys() {
-            let cid = Cid::try_from(key.as_slice())?;
-            set.insert(cid);
+        for &key in self.blockchain_db.read().keys() {
+            set.insert(key);
         }
         Ok(set)
     }
@@ -36,17 +62,11 @@ impl GarbageCollectable<CidHashSet> for MemoryDB {
         let mut db = self.blockchain_db.write();
         let mut deleted = 0;
         db.retain(|key, _| {
-            let cid = Cid::try_from(key.as_slice());
-            match cid {
-                Ok(cid) => {
-                    let retain = !keys.contains(&cid);
-                    if !retain {
-                        deleted += 1;
-                    }
-                    retain
-                }
-                _ => true,
+            let retain = !keys.contains(key);
+            if !retain {
+                deleted += 1;
             }
+            retain
         });
         Ok(deleted)
     }
@@ -111,22 +131,15 @@ impl EthMappingsStore for MemoryDB {
 
 impl Blockstore for MemoryDB {
     fn get(&self, k: &Cid) -> anyhow::Result<Option<Vec<u8>>> {
-        Ok(self
-            .blockchain_db
+        Ok(self.blockchain_db.read().get(k).cloned().or(self
+            .blockchain_persistent_db
             .read()
-            .get(&k.to_bytes())
-            .cloned()
-            .or(self
-                .blockchain_persistent_db
-                .read()
-                .get(&k.to_bytes())
-                .cloned()))
+            .get(k)
+            .cloned()))
     }
 
     fn put_keyed(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> {
-        self.blockchain_db
-            .write()
-            .insert(k.to_bytes(), block.to_vec());
+        self.blockchain_db.write().insert(*k, block.to_vec());
         Ok(())
     }
 }
@@ -135,14 +148,14 @@ impl PersistentStore for MemoryDB {
     fn put_keyed_persistent(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> {
         self.blockchain_persistent_db
             .write()
-            .insert(k.to_bytes(), block.to_vec());
+            .insert(*k, block.to_vec());
         Ok(())
     }
 }
 
 impl BitswapStoreRead for MemoryDB {
     fn contains(&self, cid: &Cid) -> anyhow::Result<bool> {
-        Ok(self.blockchain_db.read().contains_key(&cid.to_bytes()))
+        Ok(self.blockchain_db.read().contains_key(cid))
     }
 
     fn get(&self, cid: &Cid) -> anyhow::Result<Option<Vec<u8>>> {