diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs index cde88d09de..3bec4702b3 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs @@ -14,7 +14,9 @@ use mithril_common::{ CardanoNetwork, StdResult, }; -use crate::{snapshotter::OngoingSnapshot, FileUploader, LocalSnapshotUploader, Snapshotter}; +use crate::{ + file_uploaders::LocalUploader, snapshotter::OngoingSnapshot, FileUploader, Snapshotter, +}; /// The [AncillaryFileUploader] trait allows identifying uploaders that return locations for ancillary archive files. #[cfg_attr(test, mockall::automock)] @@ -25,7 +27,7 @@ pub trait AncillaryFileUploader: Send + Sync { } #[async_trait] -impl AncillaryFileUploader for LocalSnapshotUploader { +impl AncillaryFileUploader for LocalUploader { async fn upload(&self, filepath: &Path) -> StdResult { let uri = FileUploader::upload(self, filepath).await?.into(); diff --git a/mithril-aggregator/src/dependency_injection/builder.rs b/mithril-aggregator/src/dependency_injection/builder.rs index 45b3603fdc..ca3bbb8b73 100644 --- a/mithril-aggregator/src/dependency_injection/builder.rs +++ b/mithril-aggregator/src/dependency_injection/builder.rs @@ -65,7 +65,7 @@ use crate::{ }, entities::AggregatorEpochSettings, event_store::{EventMessage, EventStore, TransmitterService}, - file_uploaders::{FileUploader, GcpUploader}, + file_uploaders::{FileUploader, GcpUploader, LocalUploader}, http_server::{ routes::router::{self, RouterConfig, RouterState}, SERVER_BASE_PATH, @@ -1226,8 +1226,7 @@ impl DependenciesBuilder { message: format!("Could not parse server url:'{url}'."), error: Some(e.into()), })?; - let local_uploader = - LocalSnapshotUploader::new(server_url_prefix, &snapshot_dir, logger.clone())?; + let local_uploader = LocalUploader::new(server_url_prefix, &snapshot_dir, logger.clone())?; let ancillary_builder = Arc::new(AncillaryArtifactBuilder::new( vec![Arc::new(local_uploader)], snapshotter, diff --git a/mithril-aggregator/src/file_uploaders/local_uploader.rs b/mithril-aggregator/src/file_uploaders/local_uploader.rs new file mode 100644 index 0000000000..21e2a1157b --- /dev/null +++ b/mithril-aggregator/src/file_uploaders/local_uploader.rs @@ -0,0 +1,157 @@ +use anyhow::Context; +use async_trait::async_trait; +use reqwest::Url; +use slog::{debug, Logger}; +use std::path::{Path, PathBuf}; + +use mithril_common::logging::LoggerExtensions; +use mithril_common::StdResult; + +use crate::file_uploaders::{url_sanitizer::sanitize_url_path, FileUploader, FileUri}; + +/// LocalUploader is a file uploader working using local files +pub struct LocalUploader { + /// File server URL prefix + server_url_prefix: Url, + + /// Target folder where to store files archive + target_location: PathBuf, + + logger: Logger, +} + +impl LocalUploader { + /// LocalUploader factory + pub(crate) fn new( + server_url_prefix: Url, + target_location: &Path, + logger: Logger, + ) -> StdResult { + let logger = logger.new_with_component_name::(); + debug!(logger, "New LocalUploader created"; "server_url_prefix" => &server_url_prefix.as_str()); + let server_url_prefix = sanitize_url_path(&server_url_prefix)?; + + Ok(Self { + server_url_prefix, + target_location: target_location.to_path_buf(), + logger, + }) + } +} + +#[async_trait] +impl FileUploader for LocalUploader { + async fn upload(&self, filepath: &Path) -> StdResult { + let archive_name = filepath.file_name().unwrap().to_str().unwrap(); + let target_path = &self.target_location.join(archive_name); + tokio::fs::copy(filepath, target_path) + .await + .with_context(|| "File copy failure")?; + + let location = &self + .server_url_prefix + .join("artifact/snapshot/")? + .join(archive_name)?; + let location = location.as_str().to_string(); + + debug!(self.logger, "File 'uploaded' to local storage"; "location" => &location); + Ok(FileUri(location)) + } +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::io::Write; + use std::path::{Path, PathBuf}; + + use mithril_common::test_utils::TempDir; + + use crate::test_tools::TestLogger; + + use super::*; + + fn create_fake_archive(dir: &Path, name: &str) -> PathBuf { + let file_path = dir.join(format!("{name}.tar.gz")); + let mut file = File::create(&file_path).unwrap(); + writeln!( + file, + "I swear, this is an archive, not a temporary test file." + ) + .unwrap(); + + file_path + } + + #[tokio::test] + async fn should_extract_archive_name_to_deduce_location() { + let source_dir = TempDir::create( + "local_uploader", + "should_extract_archive_name_to_deduce_location_source", + ); + let target_dir = TempDir::create( + "local_uploader", + "should_extract_archive_name_to_deduce_location_target", + ); + let archive_name = "an_archive"; + let archive = create_fake_archive(&source_dir, archive_name); + let expected_location = format!( + "http://test.com:8080/base-root/artifact/snapshot/{}", + &archive.file_name().unwrap().to_str().unwrap() + ); + + let url_prefix = Url::parse("http://test.com:8080/base-root").unwrap(); + let uploader = LocalUploader::new(url_prefix, &target_dir, TestLogger::stdout()).unwrap(); + let location = uploader + .upload(&archive) + .await + .expect("local upload should not fail"); + + assert_eq!(FileUri(expected_location), location); + } + + #[tokio::test] + async fn should_copy_file_to_target_location() { + let source_dir = TempDir::create( + "local_uploader", + "should_copy_file_to_target_location_source", + ); + let target_dir = TempDir::create( + "local_uploader", + "should_copy_file_to_target_location_target", + ); + println!("target_dir: {:?}", target_dir); + let archive = create_fake_archive(&source_dir, "an_archive"); + let uploader = LocalUploader::new( + Url::parse("http://test.com:8080/base-root/").unwrap(), + &target_dir, + TestLogger::stdout(), + ) + .unwrap(); + uploader.upload(&archive).await.unwrap(); + + assert!(target_dir.join(archive.file_name().unwrap()).exists()); + } + + #[tokio::test] + async fn should_error_if_path_is_a_directory() { + let source_dir = TempDir::create( + "local_uploader", + "should_error_if_path_is_a_directory_source", + ); + let target_dir = TempDir::create( + "local_uploader", + "should_error_if_path_is_a_directory_target", + ); + let uploader = LocalUploader::new( + Url::parse("http://test.com:8080/base-root/").unwrap(), + &target_dir, + TestLogger::stdout(), + ) + .unwrap(); + uploader + .upload(&source_dir) + .await + .expect_err("Uploading a directory should fail"); + } +} diff --git a/mithril-aggregator/src/file_uploaders/mod.rs b/mithril-aggregator/src/file_uploaders/mod.rs index 31ddab38fd..428d385136 100644 --- a/mithril-aggregator/src/file_uploaders/mod.rs +++ b/mithril-aggregator/src/file_uploaders/mod.rs @@ -2,6 +2,7 @@ mod dumb_uploader; mod gcp_uploader; mod interface; mod local_snapshot_uploader; +mod local_uploader; pub mod url_sanitizer; pub use dumb_uploader::*; @@ -9,6 +10,7 @@ pub use gcp_uploader::GcpUploader; pub use interface::FileUploader; pub use interface::FileUri; pub use local_snapshot_uploader::LocalSnapshotUploader; +pub use local_uploader::LocalUploader; #[cfg(test)] pub use interface::MockFileUploader;