Skip to content

Commit

Permalink
feature: plugs immutable_builder to the `CardanoDatabaseArtifactBui…
Browse files Browse the repository at this point in the history 
…lder`
  • Loading branch information
@sfauvel
sfauvel committed Jan 14, 2025
1 parent e1257be commit 37592dd
Show file tree
Hide file tree
Showing 4 changed files with 229 additions and 157 deletions.
87 changes: 66 additions & 21 deletions mithril-aggregator/src/artifact_builder/cardano_database.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,14 @@ use mithril_common::{

use crate::artifact_builder::{AncillaryArtifactBuilder, ArtifactBuilder};

use super::ImmutableArtifactBuilder;

pub struct CardanoDatabaseArtifactBuilder {
db_directory: PathBuf,
cardano_node_version: Version,
compression_algorithm: CompressionAlgorithm,
ancillary_builder: Arc<AncillaryArtifactBuilder>,
immutable_builder: Arc<ImmutableArtifactBuilder>,
}

impl CardanoDatabaseArtifactBuilder {
Expand All @@ -30,12 +33,14 @@ impl CardanoDatabaseArtifactBuilder {
cardano_node_version: &Version,
compression_algorithm: CompressionAlgorithm,
ancillary_builder: Arc<AncillaryArtifactBuilder>,
immutable_builder: Arc<ImmutableArtifactBuilder>,
) -> Self {
Self {
db_directory,
cardano_node_version: cardano_node_version.clone(),
compression_algorithm,
ancillary_builder,
immutable_builder,
}
}
}
Expand All @@ -62,11 +67,15 @@ impl ArtifactBuilder<CardanoDbBeacon, CardanoDatabaseSnapshot> for CardanoDataba
let total_db_size_uncompressed = compute_uncompressed_database_size(&self.db_directory)?;

let ancillary_locations = self.ancillary_builder.upload(&beacon).await?;
let immutables_locations = self
.immutable_builder
.upload(beacon.immutable_file_number)
.await?;

let locations = ArtifactsLocations {
ancillary: ancillary_locations,
digests: vec![],
immutables: vec![],
immutables: immutables_locations,
};

let cardano_database = CardanoDatabaseSnapshot::new(
Expand Down Expand Up @@ -113,13 +122,18 @@ mod tests {

use mithril_common::{
digesters::DummyCardanoDbBuilder,
entities::{AncillaryLocation, ProtocolMessage, ProtocolMessagePartKey},
entities::{
AncillaryLocation, ImmutablesLocation, MultiFilesUri, ProtocolMessage,
ProtocolMessagePartKey, TemplateUri,
},
test_utils::{fake_data, TempDir},
CardanoNetwork,
};

use crate::{
artifact_builder::MockAncillaryFileUploader, test_tools::TestLogger, DumbSnapshotter,
artifact_builder::{MockAncillaryFileUploader, MockImmutableFilesUploader},
test_tools::TestLogger,
DumbSnapshotter,
};

use super::*;
Expand Down Expand Up @@ -155,6 +169,7 @@ mod tests {
async fn should_compute_valid_artifact() {
let test_dir = get_test_directory("should_compute_valid_artifact");

let beacon = fake_data::beacon();
let immutable_trio_file_size = 777;
let ledger_file_size = 6666;
let volatile_file_size = 99;
Expand All @@ -168,29 +183,55 @@ mod tests {
.build();
let expected_total_size = immutable_trio_file_size + ledger_file_size + volatile_file_size;

let mut ancillary_uploader = MockAncillaryFileUploader::new();
ancillary_uploader.expect_upload().return_once(|_| {
Ok(AncillaryLocation::CloudStorage {
uri: "ancillary_uri".to_string(),
})
});
let ancillary_artifact_builder = {
let mut ancillary_uploader = MockAncillaryFileUploader::new();
ancillary_uploader.expect_upload().return_once(|_| {
Ok(AncillaryLocation::CloudStorage {
uri: "ancillary_uri".to_string(),
})
});

AncillaryArtifactBuilder::new(
vec![Arc::new(ancillary_uploader)],
Arc::new(DumbSnapshotter::new()),
CardanoNetwork::DevNet(123),
CompressionAlgorithm::Gzip,
TestLogger::stdout(),
)
.unwrap()
};

let immutable_artifact_builder = {
let number_of_immutable_file_loaded = fake_data::beacon().immutable_file_number;
let mut immutable_uploader = MockImmutableFilesUploader::new();
immutable_uploader
.expect_batch_upload()
.withf(move |paths| paths.len() == number_of_immutable_file_loaded as usize)
.return_once(|_| {
Ok(ImmutablesLocation::CloudStorage {
uri: MultiFilesUri::Template(TemplateUri(
"immutable_template_uri".to_string(),
)),
})
});

ImmutableArtifactBuilder::new(
vec![Arc::new(immutable_uploader)],
Arc::new(DumbSnapshotter::new()),
CompressionAlgorithm::Gzip,
TestLogger::stdout(),
)
.unwrap()
};

let cardano_database_artifact_builder = CardanoDatabaseArtifactBuilder::new(
test_dir,
&Version::parse("1.0.0").unwrap(),
CompressionAlgorithm::Zstandard,
Arc::new(
AncillaryArtifactBuilder::new(
vec![Arc::new(ancillary_uploader)],
Arc::new(DumbSnapshotter::new()),
CardanoNetwork::DevNet(123),
CompressionAlgorithm::Gzip,
TestLogger::stdout(),
)
.unwrap(),
),
Arc::new(ancillary_artifact_builder),
Arc::new(immutable_artifact_builder),
);

let beacon = fake_data::beacon();
let certificate_with_merkle_root = {
let mut protocol_message = ProtocolMessage::new();
protocol_message.set_message_part(
Expand All @@ -211,14 +252,18 @@ mod tests {
let expected_ancillary_locations = vec![AncillaryLocation::CloudStorage {
uri: "ancillary_uri".to_string(),
}];

let expected_immutables_locations = vec![ImmutablesLocation::CloudStorage {
uri: MultiFilesUri::Template(TemplateUri("immutable_template_uri".to_string())),
}];
let artifact_expected = CardanoDatabaseSnapshot::new(
"merkleroot".to_string(),
beacon,
expected_total_size,
ArtifactsLocations {
ancillary: expected_ancillary_locations,
digests: vec![],
immutables: vec![],
immutables: expected_immutables_locations,
},
CompressionAlgorithm::Zstandard,
&Version::parse("1.0.0").unwrap(),
Expand Down
45 changes: 28 additions & 17 deletions mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use slog::{error, Logger};

use mithril_common::{
digesters::IMMUTABLE_DIR,
entities::{CompressionAlgorithm, ImmutableFileNumber, MultiFilesUri},
entities::{CompressionAlgorithm, ImmutableFileNumber, ImmutablesLocation, MultiFilesUri},
logging::LoggerExtensions,
StdResult,
};
Expand All @@ -36,12 +36,12 @@ fn default_extractor(file_uri: &str) -> StdResult<Option<String>> {
#[async_trait]
pub trait ImmutableFilesUploader: Send + Sync {
/// Uploads the archives at the given filepaths and returns the location of the uploaded file.
async fn batch_upload<'a>(&self, filepaths: &[&'a Path]) -> StdResult<MultiFilesUri>;
async fn batch_upload<'a>(&self, filepaths: &[&'a Path]) -> StdResult<ImmutablesLocation>;
}

#[async_trait]
impl ImmutableFilesUploader for LocalUploader {
async fn batch_upload<'a>(&self, filepaths: &[&'a Path]) -> StdResult<MultiFilesUri> {
async fn batch_upload<'a>(&self, filepaths: &[&'a Path]) -> StdResult<ImmutablesLocation> {
let mut file_uris = Vec::new();
for filepath in filepaths {
file_uris.push(self.upload(filepath).await?.into());
Expand All @@ -50,7 +50,9 @@ impl ImmutableFilesUploader for LocalUploader {
let template_uri = MultiFilesUri::extract_template_from_uris(file_uris, default_extractor)?
.ok_or_else(|| anyhow!("No template found in the uploaded files"))?;

Ok(MultiFilesUri::Template(template_uri))
Ok(ImmutablesLocation::CloudStorage {
uri: MultiFilesUri::Template(template_uri),
})
}
}

Expand Down Expand Up @@ -85,7 +87,7 @@ impl ImmutableArtifactBuilder {
pub async fn upload(
&self,
up_to_immutable_file_number: ImmutableFileNumber,
) -> StdResult<Vec<MultiFilesUri>> {
) -> StdResult<Vec<ImmutablesLocation>> {
let archives_paths =
self.immutable_archives_paths_creating_the_missing_ones(up_to_immutable_file_number)?;
let archives_paths: Vec<_> = archives_paths.iter().map(Path::new).collect();
Expand Down Expand Up @@ -138,7 +140,7 @@ impl ImmutableArtifactBuilder {
async fn upload_immutable_archives(
&self,
archive_paths: &[&Path],
) -> StdResult<Vec<MultiFilesUri>> {
) -> StdResult<Vec<ImmutablesLocation>> {
let mut locations = Vec::new();
for uploader in &self.uploaders {
let result = uploader.batch_upload(archive_paths).await;
Expand Down Expand Up @@ -197,7 +199,11 @@ mod tests {
equivalent_to(paths, archive_paths.clone())
})
.times(1)
.return_once(|_| Ok(MultiFilesUri::Template(TemplateUri(uri))));
.return_once(|_| {
Ok(ImmutablesLocation::CloudStorage {
uri: MultiFilesUri::Template(TemplateUri(uri)),
})
});

uploader
}
Expand Down Expand Up @@ -264,9 +270,9 @@ mod tests {

assert_equivalent(
archive_paths,
vec![MultiFilesUri::Template(TemplateUri(
"archive.tar.gz".to_string(),
))],
vec![ImmutablesLocation::CloudStorage {
uri: MultiFilesUri::Template(TemplateUri("archive.tar.gz".to_string())),
}],
)
}

Expand Down Expand Up @@ -616,9 +622,9 @@ mod tests {

assert_equivalent(
archive_paths,
vec![MultiFilesUri::Template(TemplateUri(
"archive_2.tar.gz".to_string(),
))],
vec![ImmutablesLocation::CloudStorage {
uri: MultiFilesUri::Template(TemplateUri("archive_2.tar.gz".to_string())),
}],
)
}

Expand Down Expand Up @@ -651,8 +657,12 @@ mod tests {
assert_equivalent(
archive_paths,
vec![
MultiFilesUri::Template(TemplateUri("archive_1.tar.gz".to_string())),
MultiFilesUri::Template(TemplateUri("archive_2.tar.gz".to_string())),
ImmutablesLocation::CloudStorage {
uri: MultiFilesUri::Template(TemplateUri("archive_1.tar.gz".to_string())),
},
ImmutablesLocation::CloudStorage {
uri: MultiFilesUri::Template(TemplateUri("archive_2.tar.gz".to_string())),
},
],
)
}
Expand Down Expand Up @@ -704,10 +714,11 @@ mod tests {
assert!(target_dir.join(archive_1.file_name().unwrap()).exists());
assert!(target_dir.join(archive_2.file_name().unwrap()).exists());

let expected_location = MultiFilesUri::Template(TemplateUri(
let expected_location = ImmutablesLocation::CloudStorage {
uri: MultiFilesUri::Template(TemplateUri(
"http://test.com:8080/base-root/artifact/snapshot/{immutable_file_number}.tar.gz"
.to_string(),
));
))};
assert_eq!(expected_location, location);
}

Expand Down
15 changes: 13 additions & 2 deletions mithril-aggregator/src/dependency_injection/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ use crate::{
artifact_builder::{
AncillaryArtifactBuilder, CardanoDatabaseArtifactBuilder,
CardanoImmutableFilesFullArtifactBuilder, CardanoStakeDistributionArtifactBuilder,
CardanoTransactionsArtifactBuilder, MithrilStakeDistributionArtifactBuilder,
CardanoTransactionsArtifactBuilder, ImmutableArtifactBuilder,
MithrilStakeDistributionArtifactBuilder,
},
configuration::ExecutionEnvironment,
database::repository::{
Expand Down Expand Up @@ -1246,17 +1247,27 @@ impl DependenciesBuilder {
LocalUploader::new(self.get_server_url_prefix()?, &snapshot_dir, logger.clone())?;
let ancillary_builder = Arc::new(AncillaryArtifactBuilder::new(
vec![Arc::new(local_uploader)],
snapshotter,
snapshotter.clone(),
self.configuration.get_network()?,
self.configuration.snapshot_compression_algorithm,
logger.clone(),
)?);

let local_uploader =
LocalUploader::new(self.get_server_url_prefix()?, &snapshot_dir, logger.clone())?;
let immutable_builder = Arc::new(ImmutableArtifactBuilder::new(
vec![Arc::new(local_uploader)],
snapshotter,
self.configuration.snapshot_compression_algorithm,
logger.clone(),
)?);

Ok(CardanoDatabaseArtifactBuilder::new(
self.configuration.db_directory.clone(),
&cardano_node_version,
self.configuration.snapshot_compression_algorithm,
ancillary_builder,
immutable_builder,
))
}

Expand Down
Loading

0 comments on commit 37592dd

Please sign in to comment.