diff --git a/mithril-aggregator/src/artifact_builder/cardano_database.rs b/mithril-aggregator/src/artifact_builder/cardano_database.rs index 7cd4333aa6..862ee1db59 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database.rs @@ -122,7 +122,7 @@ fn compute_uncompressed_database_size(path: &Path) -> StdResult { #[cfg(test)] mod tests { - use std::path::PathBuf; + use std::{collections::BTreeMap, path::PathBuf}; use mithril_common::{ digesters::DummyCardanoDbBuilder, @@ -136,9 +136,8 @@ mod tests { use reqwest::Url; use crate::{ - artifact_builder::{ - MockAncillaryFileUploader, MockDigestFileUploader, MockImmutableFilesUploader, - }, + artifact_builder::{MockAncillaryFileUploader, MockImmutableFilesUploader}, + immutable_file_digest_mapper::MockImmutableFileDigestMapper, test_tools::TestLogger, DumbSnapshotter, }; @@ -231,12 +230,21 @@ mod tests { .unwrap() }; - let digest_artifact_builder = DigestArtifactBuilder::new( - Url::parse("http://aggregator_uri").unwrap(), - vec![], - TestLogger::stdout(), - ) - .unwrap(); + let digest_artifact_builder = { + let mut immutable_file_digest_mapper = MockImmutableFileDigestMapper::new(); + + immutable_file_digest_mapper + .expect_get_immutable_file_digest_map() + .returning(|| Ok(BTreeMap::new())); + + DigestArtifactBuilder::new( + Url::parse("http://aggregator_uri").unwrap(), + vec![], + Arc::new(immutable_file_digest_mapper), + TestLogger::stdout(), + ) + .unwrap() + }; let cardano_database_artifact_builder = CardanoDatabaseArtifactBuilder::new( test_dir, diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs index c3e481b649..b7590f0518 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs @@ -1,15 +1,19 @@ use std::{ + fs, path::{Path, PathBuf}, sync::Arc, }; -use anyhow::anyhow; +use anyhow::Context; use async_trait::async_trait; -use mithril_common::{entities::DigestLocation, logging::LoggerExtensions, StdResult}; +use mithril_common::{ + entities::DigestLocation, logging::LoggerExtensions, + messages::CardanoDatabaseDigestListItemMessage, StdResult, +}; use reqwest::Url; -use slog::{debug, error, Logger}; +use slog::{error, Logger}; -use crate::{file_uploaders::url_sanitizer::sanitize_url_path, snapshotter::OngoingSnapshot}; +use crate::ImmutableFileDigestMapper; /// The [DigestFileUploader] trait allows identifying uploaders that return locations for digest archive files. #[cfg_attr(test, mockall::automock)] @@ -24,6 +28,9 @@ pub struct DigestArtifactBuilder { aggregator_url_prefix: Url, /// Uploaders uploaders: Vec>, + + immutable_file_digest_mapper: Arc, + logger: Logger, } @@ -32,27 +39,60 @@ impl DigestArtifactBuilder { pub fn new( aggregator_url_prefix: Url, uploaders: Vec>, + immutable_file_digest_mapper: Arc, logger: Logger, ) -> StdResult { Ok(Self { aggregator_url_prefix, uploaders, + immutable_file_digest_mapper, logger: logger.new_with_component_name::(), }) } pub async fn upload(&self) -> StdResult> { - // let snapshot = self.create_digest_archive(beacon)?; - // let digest_path = snapshot.get_file_path(); - let digest_path = Path::new(""); + let digest_path = self.create_digest_archive().await?; - self.upload_digest_archive(digest_path).await + let locations = self.upload_digest_archive(&digest_path).await; + fs::remove_file(&digest_path).with_context(|| { + format!( + "Could not remove digest archive file: '{}'", + digest_path.display() + ) + })?; + locations } - async fn create_digest_archive() -> StdResult { - // get message service::get_cardano_database_digest_list_message - // output json (created from message) to file - todo!() + async fn create_digest_archive(&self) -> StdResult { + let immutable_file_digest_map = self + .immutable_file_digest_mapper + .get_immutable_file_digest_map() + .await? + .into_iter() + .map( + |(immutable_file_name, digest)| CardanoDatabaseDigestListItemMessage { + immutable_file_name, + digest, + }, + ) + .collect::>(); + + // TODO : change that injecting the path or using snapshotter + let digests_file_path = Path::new("/tmp").join("mithril").join("digests.json"); + + if let Some(digests_dir) = digests_file_path.parent() { + fs::create_dir_all(digests_dir).with_context(|| { + format!( + "Can not create digests directory: '{}'", + digests_dir.display() + ) + })?; + } + + let digest_file = fs::File::create(digests_file_path.clone()).unwrap(); + serde_json::to_writer(digest_file, &immutable_file_digest_map)?; + + Ok(digests_file_path) } /// Uploads the digest archive and returns the locations of the uploaded files. @@ -94,8 +134,16 @@ impl DigestArtifactBuilder { #[cfg(test)] mod tests { - use crate::test_tools::TestLogger; - use mithril_common::test_utils::{assert_equivalent, TempDir}; + use std::{collections::BTreeMap, fs::read_to_string}; + + use crate::{ + immutable_file_digest_mapper::MockImmutableFileDigestMapper, test_tools::TestLogger, + }; + use anyhow::anyhow; + use mithril_common::{ + messages::{CardanoDatabaseDigestListItemMessage, CardanoDatabaseDigestListMessage}, + test_utils::{assert_equivalent, TempDir}, + }; use super::*; @@ -121,9 +169,15 @@ mod tests { #[tokio::test] async fn digest_artifact_builder_return_digests_route_on_aggregator() { + let mut immutable_file_digest_mapper = MockImmutableFileDigestMapper::new(); + immutable_file_digest_mapper + .expect_get_immutable_file_digest_map() + .returning(|| Ok(BTreeMap::new())); + let builder = DigestArtifactBuilder::new( Url::parse("https://aggregator/").unwrap(), vec![], + Arc::new(immutable_file_digest_mapper), TestLogger::stdout(), ) .unwrap(); @@ -151,11 +205,14 @@ mod tests { let builder = DigestArtifactBuilder::new( Url::parse("https://aggregator/").unwrap(), vec![Arc::new(uploader)], + Arc::new(MockImmutableFileDigestMapper::new()), TestLogger::file(&log_path), ) .unwrap(); - let _ = builder.upload_digest_archive(&Path::new("")).await; + let _ = builder + .upload_digest_archive(Path::new("digest_file")) + .await; } let logs = std::fs::read_to_string(&log_path).unwrap(); @@ -169,11 +226,15 @@ mod tests { let builder = DigestArtifactBuilder::new( Url::parse("https://aggregator/").unwrap(), vec![Arc::new(uploader)], + Arc::new(MockImmutableFileDigestMapper::new()), TestLogger::stdout(), ) .unwrap(); - let locations = builder.upload_digest_archive(&Path::new("")).await.unwrap(); + let locations = builder + .upload_digest_archive(Path::new("digest_file")) + .await + .unwrap(); assert!(!locations.is_empty()); } @@ -193,11 +254,15 @@ mod tests { let builder = DigestArtifactBuilder::new( Url::parse("https://aggregator/").unwrap(), uploaders, + Arc::new(MockImmutableFileDigestMapper::new()), TestLogger::stdout(), ) .unwrap(); - let locations = builder.upload_digest_archive(&Path::new("")).await.unwrap(); + let locations = builder + .upload_digest_archive(Path::new("digest_file")) + .await + .unwrap(); assert_equivalent( locations, @@ -223,11 +288,15 @@ mod tests { let builder = DigestArtifactBuilder::new( Url::parse("https://aggregator/").unwrap(), uploaders, + Arc::new(MockImmutableFileDigestMapper::new()), TestLogger::stdout(), ) .unwrap(); - let locations = builder.upload_digest_archive(&Path::new("")).await.unwrap(); + let locations = builder + .upload_digest_archive(Path::new("digest_file")) + .await + .unwrap(); assert_equivalent( locations, @@ -244,4 +313,71 @@ mod tests { ], ); } + + #[tokio::test] + async fn create_digest_archive_should_create_json_file_with_all_digests() { + let mut immutable_file_digest_mapper = MockImmutableFileDigestMapper::new(); + immutable_file_digest_mapper + .expect_get_immutable_file_digest_map() + .returning(|| { + Ok(BTreeMap::from([( + "06685.chunk".to_string(), + "0af556ab2620dd9363bf76963a231abe8948a500ea6be31b131d87907ab09b1e".to_string(), + )])) + }); + + let builder = DigestArtifactBuilder::new( + Url::parse("https://aggregator/").unwrap(), + vec![], + Arc::new(immutable_file_digest_mapper), + TestLogger::stdout(), + ) + .unwrap(); + + let archive_path = builder.create_digest_archive().await.unwrap(); + let file_content = read_to_string(archive_path).unwrap(); + let digest_content: CardanoDatabaseDigestListMessage = + serde_json::from_str(&file_content).unwrap(); + + assert_eq!( + digest_content, + vec![CardanoDatabaseDigestListItemMessage { + immutable_file_name: "06685.chunk".to_string(), + digest: "0af556ab2620dd9363bf76963a231abe8948a500ea6be31b131d87907ab09b1e" + .to_string(), + }] + ); + } + + #[tokio::test] + async fn upload_should_call_upload_with_created_digest_file_and_delete_the_file() { + // TODO : This test is flaky because we create and remove a file with an hard coded path + let mut immutable_file_digest_mapper = MockImmutableFileDigestMapper::new(); + immutable_file_digest_mapper + .expect_get_immutable_file_digest_map() + .returning(|| Ok(BTreeMap::new())); + + let mut digest_file_uploader = MockDigestFileUploader::new(); + digest_file_uploader + .expect_upload() + .withf(|path| path == Path::new("/tmp/mithril/digests.json") && path.exists()) + .times(1) + .return_once(|_| { + Ok(DigestLocation::CloudStorage { + uri: "an_uri".to_string(), + }) + }); + + let builder = DigestArtifactBuilder::new( + Url::parse("https://aggregator/").unwrap(), + vec![Arc::new(digest_file_uploader)], + Arc::new(immutable_file_digest_mapper), + TestLogger::stdout(), + ) + .unwrap(); + + let _locations = builder.upload().await.unwrap(); + + assert!(!Path::new("/tmp/mithril/digests.json").exists()); + } } diff --git a/mithril-aggregator/src/dependency_injection/builder.rs b/mithril-aggregator/src/dependency_injection/builder.rs index a904743ec8..ba6d301a02 100644 --- a/mithril-aggregator/src/dependency_injection/builder.rs +++ b/mithril-aggregator/src/dependency_injection/builder.rs @@ -1230,6 +1230,7 @@ impl DependenciesBuilder { logger: &Logger, cardano_node_version: Version, snapshotter: Arc, + immutable_file_digest_mapper: Arc, ) -> Result { let artifacts_dir = Path::new("cardano-database").join("ancillary"); let snapshot_dir = self @@ -1265,6 +1266,7 @@ impl DependenciesBuilder { let digest_builder = Arc::new(DigestArtifactBuilder::new( self.get_server_url_prefix()?, vec![], + immutable_file_digest_mapper, logger.clone(), )?); @@ -1305,11 +1307,13 @@ impl DependenciesBuilder { let stake_store = self.get_stake_store().await?; let cardano_stake_distribution_artifact_builder = Arc::new(CardanoStakeDistributionArtifactBuilder::new(stake_store)); + let immutable_file_digest_mapper = self.get_immutable_file_digest_mapper().await?; let cardano_database_artifact_builder = Arc::new(self.create_cardano_database_artifact_builder( &logger, cardano_node_version, snapshotter, + immutable_file_digest_mapper, )?); let dependencies = SignedEntityServiceArtifactsDependencies::new( mithril_stake_distribution_artifact_builder, @@ -1825,7 +1829,9 @@ impl DependenciesBuilder { mod tests { use mithril_common::{entities::SignedEntityTypeDiscriminants, test_utils::TempDir}; - use crate::test_tools::TestLogger; + use crate::{ + immutable_file_digest_mapper::MockImmutableFileDigestMapper, test_tools::TestLogger, + }; use super::*; @@ -1888,11 +1894,14 @@ mod tests { assert!(!ancillary_dir.exists()); + let immutable_file_digest_mapper = MockImmutableFileDigestMapper::new(); + dep_builder .create_cardano_database_artifact_builder( &TestLogger::stdout(), Version::parse("1.0.0").unwrap(), Arc::new(DumbSnapshotter::new()), + Arc::new(immutable_file_digest_mapper), ) .unwrap();