Feat: Create immutable builder for incremental Cardano DB

CHANGELOG.md

@@ -18,6 +18,8 @@ As a minor extension, we have adopted a slightly different versioning convention
   - Implement the artifact routes of the aggregator for the signed entity type `CardanoDatabase`.
   - Implement the immutable file digests route in the aggregator.
   - Implement the artifact ancillary builder in the aggregator.
+  - Implement the artifact immutable builder in the aggregator.
+  - Implement the artifact digest builder in the aggregator.
 
 - Crates versions:
 

mithril-aggregator/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.6.12"
+version = "0.6.13"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }
@@ -35,6 +35,7 @@ openssl-probe = { version = "0.1.5", optional = true }
 paste = "1.0.15"
 prometheus = "0.13.4"
 rayon = "1.10.0"
+regex = "1.11.1"
 reqwest = { version = "0.12.12", features = ["json"] }
 semver = "1.0.24"
 serde = { version = "1.0.217", features = ["derive"] }

mithril-aggregator/src/artifact_builder/cardano_database.rs

@@ -17,11 +17,15 @@ use mithril_common::{
 
 use crate::artifact_builder::{AncillaryArtifactBuilder, ArtifactBuilder};
 
+use super::{DigestArtifactBuilder, ImmutableArtifactBuilder};
+
 pub struct CardanoDatabaseArtifactBuilder {
     db_directory: PathBuf,
     cardano_node_version: Version,
     compression_algorithm: CompressionAlgorithm,
     ancillary_builder: Arc<AncillaryArtifactBuilder>,
+    immutable_builder: Arc<ImmutableArtifactBuilder>,
+    digest_builder: Arc<DigestArtifactBuilder>,
 }
 
 impl CardanoDatabaseArtifactBuilder {
@@ -30,12 +34,16 @@ impl CardanoDatabaseArtifactBuilder {
         cardano_node_version: &Version,
         compression_algorithm: CompressionAlgorithm,
         ancillary_builder: Arc<AncillaryArtifactBuilder>,
+        immutable_builder: Arc<ImmutableArtifactBuilder>,
+        digest_builder: Arc<DigestArtifactBuilder>,
     ) -> Self {
         Self {
             db_directory,
             cardano_node_version: cardano_node_version.clone(),
             compression_algorithm,
             ancillary_builder,
+            immutable_builder,
+            digest_builder,
         }
     }
 }
@@ -62,11 +70,16 @@ impl ArtifactBuilder<CardanoDbBeacon, CardanoDatabaseSnapshot> for CardanoDataba
         let total_db_size_uncompressed = compute_uncompressed_database_size(&self.db_directory)?;
 
         let ancillary_locations = self.ancillary_builder.upload(&beacon).await?;
+        let immutables_locations = self
+            .immutable_builder
+            .upload(beacon.immutable_file_number)
+            .await?;
+        let digest_locations = self.digest_builder.upload().await?;
 
         let locations = ArtifactsLocations {
             ancillary: ancillary_locations,
-            digests: vec![],
-            immutables: vec![],
+            digests: digest_locations,
+            immutables: immutables_locations,
         };
 
         let cardano_database = CardanoDatabaseSnapshot::new(
@@ -109,17 +122,24 @@ fn compute_uncompressed_database_size(path: &Path) -> StdResult<u64> {
 
 #[cfg(test)]
 mod tests {
-    use std::path::PathBuf;
+    use std::{collections::BTreeMap, path::PathBuf};
 
     use mithril_common::{
         digesters::DummyCardanoDbBuilder,
-        entities::{AncillaryLocation, ProtocolMessage, ProtocolMessagePartKey},
+        entities::{
+            AncillaryLocation, DigestLocation, ImmutablesLocation, MultiFilesUri, ProtocolMessage,
+            ProtocolMessagePartKey, TemplateUri,
+        },
         test_utils::{fake_data, TempDir},
         CardanoNetwork,
     };
+    use reqwest::Url;
 
     use crate::{
-        artifact_builder::MockAncillaryFileUploader, test_tools::TestLogger, DumbSnapshotter,
+        artifact_builder::{MockAncillaryFileUploader, MockImmutableFilesUploader},
+        immutable_file_digest_mapper::MockImmutableFileDigestMapper,
+        test_tools::TestLogger,
+        DumbSnapshotter,
     };
 
     use super::*;
@@ -155,6 +175,7 @@ mod tests {
     async fn should_compute_valid_artifact() {
         let test_dir = get_test_directory("should_compute_valid_artifact");
 
+        let beacon = fake_data::beacon();
         let immutable_trio_file_size = 777;
         let ledger_file_size = 6666;
         let volatile_file_size = 99;
@@ -168,29 +189,73 @@ mod tests {
             .build();
         let expected_total_size = immutable_trio_file_size + ledger_file_size + volatile_file_size;
 
-        let mut ancillary_uploader = MockAncillaryFileUploader::new();
-        ancillary_uploader.expect_upload().return_once(|_| {
-            Ok(AncillaryLocation::CloudStorage {
-                uri: "ancillary_uri".to_string(),
-            })
-        });
+        let ancillary_artifact_builder = {
+            let mut ancillary_uploader = MockAncillaryFileUploader::new();
+            ancillary_uploader.expect_upload().return_once(|_| {
+                Ok(AncillaryLocation::CloudStorage {
+                    uri: "ancillary_uri".to_string(),
+                })
+            });
+
+            AncillaryArtifactBuilder::new(
+                vec![Arc::new(ancillary_uploader)],
+                Arc::new(DumbSnapshotter::new()),
+                CardanoNetwork::DevNet(123),
+                CompressionAlgorithm::Gzip,
+                TestLogger::stdout(),
+            )
+            .unwrap()
+        };
+
+        let immutable_artifact_builder = {
+            let number_of_immutable_file_loaded = fake_data::beacon().immutable_file_number;
+            let mut immutable_uploader = MockImmutableFilesUploader::new();
+            immutable_uploader
+                .expect_batch_upload()
+                .withf(move |paths| paths.len() == number_of_immutable_file_loaded as usize)
+                .return_once(|_| {
+                    Ok(ImmutablesLocation::CloudStorage {
+                        uri: MultiFilesUri::Template(TemplateUri(
+                            "immutable_template_uri".to_string(),
+                        )),
+                    })
+                });
+
+            ImmutableArtifactBuilder::new(
+                vec![Arc::new(immutable_uploader)],
+                Arc::new(DumbSnapshotter::new()),
+                CompressionAlgorithm::Gzip,
+                TestLogger::stdout(),
+            )
+            .unwrap()
+        };
+
+        let digest_artifact_builder = {
+            let mut immutable_file_digest_mapper = MockImmutableFileDigestMapper::new();
+
+            immutable_file_digest_mapper
+                .expect_get_immutable_file_digest_map()
+                .returning(|| Ok(BTreeMap::new()));
+
+            DigestArtifactBuilder::new(
+                Url::parse("http://aggregator_uri").unwrap(),
+                vec![],
+                test_dir.join("digests"),
+                Arc::new(immutable_file_digest_mapper),
+                TestLogger::stdout(),
+            )
+            .unwrap()
+        };
+
         let cardano_database_artifact_builder = CardanoDatabaseArtifactBuilder::new(
             test_dir,
             &Version::parse("1.0.0").unwrap(),
             CompressionAlgorithm::Zstandard,
-            Arc::new(
-                AncillaryArtifactBuilder::new(
-                    vec![Arc::new(ancillary_uploader)],
-                    Arc::new(DumbSnapshotter::new()),
-                    CardanoNetwork::DevNet(123),
-                    CompressionAlgorithm::Gzip,
-                    TestLogger::stdout(),
-                )
-                .unwrap(),
-            ),
+            Arc::new(ancillary_artifact_builder),
+            Arc::new(immutable_artifact_builder),
+            Arc::new(digest_artifact_builder),
         );
 
-        let beacon = fake_data::beacon();
         let certificate_with_merkle_root = {
             let mut protocol_message = ProtocolMessage::new();
             protocol_message.set_message_part(
@@ -211,14 +276,23 @@ mod tests {
         let expected_ancillary_locations = vec![AncillaryLocation::CloudStorage {
             uri: "ancillary_uri".to_string(),
         }];
+
+        let expected_immutables_locations = vec![ImmutablesLocation::CloudStorage {
+            uri: MultiFilesUri::Template(TemplateUri("immutable_template_uri".to_string())),
+        }];
+
+        let expected_digest_locations = vec![DigestLocation::Aggregator {
+            uri: "http://aggregator_uri/artifact/cardano-database/digests".to_string(),
+        }];
+
         let artifact_expected = CardanoDatabaseSnapshot::new(
             "merkleroot".to_string(),
             beacon,
             expected_total_size,
             ArtifactsLocations {
                 ancillary: expected_ancillary_locations,
-                digests: vec![],
-                immutables: vec![],
+                digests: expected_digest_locations,
+                immutables: expected_immutables_locations,
             },
             CompressionAlgorithm::Zstandard,
             &Version::parse("1.0.0").unwrap(),

mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs

@@ -183,8 +183,9 @@ mod tests {
 
     use mithril_common::{
         digesters::{DummyCardanoDbBuilder, IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR},
-        test_utils::TempDir,
+        test_utils::{assert_equivalent, TempDir},
     };
+    use uuid::Uuid;
 
     use crate::{
         test_tools::TestLogger, CompressedArchiveSnapshotter, DumbSnapshotter,
@@ -309,11 +310,11 @@ mod tests {
             .await
             .unwrap();
 
-        assert_eq!(
+        assert_equivalent(
             locations,
             vec![AncillaryLocation::CloudStorage {
-                uri: "an_uri".to_string()
-            }]
+                uri: "an_uri".to_string(),
+            }],
         );
     }
 
@@ -339,22 +340,22 @@ mod tests {
             .await
             .unwrap();
 
-        assert_eq!(
+        assert_equivalent(
             locations,
             vec![
                 AncillaryLocation::CloudStorage {
-                    uri: "an_uri".to_string()
+                    uri: "an_uri".to_string(),
                 },
                 AncillaryLocation::CloudStorage {
-                    uri: "another_uri".to_string()
-                }
-            ]
+                    uri: "another_uri".to_string(),
+                },
+            ],
         );
     }
 
     #[tokio::test]
     async fn create_archive_should_embed_ledger_volatile_directories_and_last_immutables() {
-        let test_dir = "cardano_database/create_archive";
+        let test_dir = "create_archive/cardano_database";
         let cardano_db = DummyCardanoDbBuilder::new(test_dir)
             .with_immutables(&[1, 2, 3])
             .with_ledger_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
@@ -363,15 +364,14 @@ mod tests {
         std::fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
 
         let db_directory = cardano_db.get_dir().to_path_buf();
-        let snapshotter = {
-            CompressedArchiveSnapshotter::new(
-                db_directory.clone(),
-                db_directory.parent().unwrap().join("snapshot_dest"),
-                SnapshotterCompressionAlgorithm::Gzip,
-                TestLogger::stdout(),
-            )
-            .unwrap()
-        };
+        let mut snapshotter = CompressedArchiveSnapshotter::new(
+            db_directory.clone(),
+            db_directory.parent().unwrap().join("snapshot_dest"),
+            SnapshotterCompressionAlgorithm::Gzip,
+            TestLogger::stdout(),
+        )
+        .unwrap();
+        snapshotter.set_sub_temp_dir(Uuid::new_v4().to_string());
 
         let builder = AncillaryArtifactBuilder::new(
             vec![Arc::new(MockAncillaryFileUploader::new())],
@@ -422,15 +422,14 @@ mod tests {
 
     #[tokio::test]
     async fn upload_should_return_error_and_not_upload_when_archive_creation_fails() {
-        let snapshotter = {
-            CompressedArchiveSnapshotter::new(
-                PathBuf::from("directory_not_existing"),
-                PathBuf::from("whatever"),
-                SnapshotterCompressionAlgorithm::Gzip,
-                TestLogger::stdout(),
-            )
-            .unwrap()
-        };
+        let mut snapshotter = CompressedArchiveSnapshotter::new(
+            PathBuf::from("directory_not_existing"),
+            PathBuf::from("whatever"),
+            SnapshotterCompressionAlgorithm::Gzip,
+            TestLogger::stdout(),
+        )
+        .unwrap();
+        snapshotter.set_sub_temp_dir(Uuid::new_v4().to_string());
 
         let mut uploader = MockAncillaryFileUploader::new();
         uploader.expect_upload().never();