Merge pull request #2239 from input-output-hk/ensemble/2234/increment…

…al-db-artifact-production-optimizations

Incremental Cardano DB artifact production enhancements

docs/website/root/manual/develop/nodes/mithril-aggregator.md

@@ -434,6 +434,7 @@ Here is a list of the available parameters:
 | ---------------------------------------------------------------- | ------------------------------------------------------------------ | :------------------: | --------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------- | ----------------------------------------------------------------------------- | :---------------------------------------------: |
 | `server_ip`                                                      | `--server-ip`                                                      |          -           | `SERVER_IP`                                                                                               | Listening server IP                                                                                                                                   | `0.0.0.0`                                     | -                                                                             |               :heavy_check_mark:                |
 | `server_port`                                                    | `--server-port`                                                    |          -           | `SERVER_PORT`                                                                                             | Listening server port                                                                                                                                 | `8080`                                        | -                                                                             |               :heavy_check_mark:                |
+| `public_server_url`                                              | -                                                                  |          -           | `PUBLIC_SERVER_URL`                                                                                       | Public URL of the aggregator                                                                                                                          | -                                             | `https://aggregator.release-mainnet.api.mithril.network/aggregator`           |                        -                        |
 | `snapshot_directory`                                             | `--snapshot-directory`                                             |          -           | `SNAPSHOT_DIRECTORY`                                                                                      | Directory to store local snapshots of the **Cardano node**                                                                                            | `.`                                           | -                                                                             |               :heavy_check_mark:                |
 | `snapshot_store_type`                                            | -                                                                  |          -           | `SNAPSHOT_STORE_TYPE`                                                                                     | Type of snapshot store to use                                                                                                                         | -                                             | `gcp` or `local`                                                              |               :heavy_check_mark:                |
 | `snapshot_uploader_type`                                         | -                                                                  |          -           | `SNAPSHOT_UPLOADER_TYPE`                                                                                  | Type of snapshot uploader to use                                                                                                                      | -                                             | `gcp` or `local`                                                              |               :heavy_check_mark:                |

mithril-aggregator/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.6.16"
+version = "0.6.17"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs

@@ -5,7 +5,7 @@ use std::{
 
 use anyhow::{anyhow, Context};
 use async_trait::async_trait;
-use slog::{debug, error, Logger};
+use slog::{debug, error, warn, Logger};
 
 use mithril_common::{
     digesters::{IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR},
@@ -181,6 +181,13 @@ impl AncillaryArtifactBuilder {
             }
         }
 
+        if let Err(error) = tokio::fs::remove_file(archive_filepath).await {
+            warn!(
+                self.logger, " > Post upload ancillary archive file removal failure";
+                "error" => error
+            );
+        }
+
         if locations.is_empty() {
             return Err(anyhow!(
                 "Failed to upload ancillary archive with all uploaders"
@@ -233,6 +240,21 @@ mod tests {
         uploader
     }
 
+    fn create_fake_archive(dir: &Path, name: &str) -> PathBuf {
+        use std::fs::File;
+        use std::io::Write;
+
+        let file_path = dir.join(name);
+        let mut file = File::create(&file_path).unwrap();
+        writeln!(
+            file,
+            "I swear, this is an archive, not a temporary test file."
+        )
+        .unwrap();
+
+        file_path
+    }
+
     #[test]
     fn create_ancillary_builder_should_error_when_no_uploader() {
         let result = AncillaryArtifactBuilder::new(
@@ -370,6 +392,59 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn upload_ancillary_archive_should_remove_archive_after_upload() {
+        let source_dir = TempDir::create(
+            "ancillary",
+            "upload_ancillary_archive_should_remove_archive_after_upload",
+        );
+        let archive = create_fake_archive(&source_dir, "ancillary.tar.gz");
+        let uploader = fake_uploader(archive.as_os_str().to_str().unwrap(), "an_uri");
+
+        let builder = AncillaryArtifactBuilder::new(
+            vec![Arc::new(uploader)],
+            Arc::new(DumbSnapshotter::new()),
+            CardanoNetwork::DevNet(123),
+            CompressionAlgorithm::Gzip,
+            TestLogger::stdout(),
+        )
+        .unwrap();
+
+        assert!(archive.exists());
+
+        builder.upload_ancillary_archive(&archive).await.unwrap();
+
+        assert!(!archive.exists());
+    }
+
+    #[tokio::test]
+    async fn upload_ancillary_archive_should_remove_archive_when_no_uploader_succeed() {
+        let source_dir = TempDir::create(
+            "ancillary",
+            "upload_ancillary_archive_should_remove_archive_when_no_uploader_succeed",
+        );
+        let archive = create_fake_archive(&source_dir, "ancillary.tar.gz");
+        let uploader = fake_uploader_returning_error();
+
+        let builder = AncillaryArtifactBuilder::new(
+            vec![Arc::new(uploader)],
+            Arc::new(DumbSnapshotter::new()),
+            CardanoNetwork::DevNet(123),
+            CompressionAlgorithm::Gzip,
+            TestLogger::stdout(),
+        )
+        .unwrap();
+
+        assert!(archive.exists());
+
+        builder
+            .upload_ancillary_archive(&archive)
+            .await
+            .unwrap_err();
+
+        assert!(!archive.exists());
+    }
+
     #[tokio::test]
     async fn create_archive_should_embed_ledger_volatile_directories_and_last_immutables() {
         let test_dir = "create_archive/cardano_database";

mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs

@@ -780,9 +780,9 @@ mod tests {
 
             let expected_location = ImmutablesLocation::CloudStorage {
                 uri: MultiFilesUri::Template(TemplateUri(
-                "http://test.com:8080/base-root/artifact/snapshot/{immutable_file_number}.tar.gz"
-                    .to_string(),
-            ))};
+                    "http://test.com:8080/base-root/{immutable_file_number}.tar.gz".to_string(),
+                )),
+            };
             assert_eq!(expected_location, location);
         }
 

mithril-aggregator/src/configuration.rs

@@ -4,6 +4,7 @@ use mithril_common::chain_observer::ChainObserverType;
 use mithril_common::crypto_helper::ProtocolGenesisSigner;
 use mithril_common::era::adapters::EraReaderAdapterType;
 use mithril_doc::{Documenter, DocumenterDefault, StructDoc};
+use reqwest::Url;
 use serde::{Deserialize, Serialize};
 use std::collections::{BTreeSet, HashMap};
 use std::path::PathBuf;
@@ -16,6 +17,9 @@ use mithril_common::entities::{
 };
 use mithril_common::{CardanoNetwork, StdResult};
 
+use crate::http_server::SERVER_BASE_PATH;
+use crate::tools::url_sanitizer;
+
 /// Different kinds of execution environments
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub enum ExecutionEnvironment {
@@ -96,6 +100,9 @@ pub struct Configuration {
     /// Server listening port
     pub server_port: u16,
 
+    /// Server URL that can be accessed from the outside
+    pub public_server_url: Option<String>,
+
     /// Run Interval is the interval between two runtime cycles in ms
     #[example = "`60000`"]
     pub run_interval: u64,
@@ -242,6 +249,7 @@ impl Configuration {
             snapshot_use_cdn_domain: false,
             server_ip: "0.0.0.0".to_string(),
             server_port: 8000,
+            public_server_url: None,
             run_interval: 5000,
             db_directory: PathBuf::new(),
             snapshot_directory: PathBuf::new(),
@@ -273,9 +281,23 @@ impl Configuration {
         }
     }
 
-    /// Build the server URL from configuration.
-    pub fn get_server_url(&self) -> String {
-        format!("http://{}:{}/", self.server_ip, self.server_port)
+    /// Build the local server URL from configuration.
+    pub fn get_local_server_url(&self) -> StdResult<Url> {
+        let url = Url::parse(&format!(
+            "http://{}:{}/{SERVER_BASE_PATH}/",
+            self.server_ip, self.server_port
+        ))?;
+        Ok(url)
+    }
+
+    /// Get the server URL from the configuration.
+    ///
+    /// Will return the public server URL if it is set, otherwise the local server URL.
+    pub fn get_server_url(&self) -> StdResult<Url> {
+        match &self.public_server_url {
+            Some(url) => Ok(url_sanitizer::sanitize_url_path(&Url::parse(url)?)?),
+            None => self.get_local_server_url(),
+        }
     }
 
     /// Check configuration and return a representation of the Cardano network.
@@ -625,4 +647,71 @@ mod test {
 
         assert!(!config.allow_http_serve_directory());
     }
+
+    #[test]
+    fn get_server_url_return_local_url_with_server_base_path_if_public_url_is_not_set() {
+        let config = Configuration {
+            server_ip: "1.2.3.4".to_string(),
+            server_port: 5678,
+            public_server_url: None,
+            ..Configuration::new_sample()
+        };
+
+        assert_eq!(
+            config.get_server_url().unwrap().as_str(),
+            &format!("http://1.2.3.4:5678/{SERVER_BASE_PATH}/")
+        );
+    }
+
+    #[test]
+    fn get_server_url_return_sanitized_public_url_if_it_is_set() {
+        let config = Configuration {
+            server_ip: "1.2.3.4".to_string(),
+            server_port: 5678,
+            public_server_url: Some("https://example.com".to_string()),
+            ..Configuration::new_sample()
+        };
+
+        assert_eq!(
+            config.get_server_url().unwrap().as_str(),
+            "https://example.com/"
+        );
+    }
+
+    #[test]
+    fn joining_to_local_server_url_keep_base_path() {
+        let config = Configuration {
+            server_ip: "1.2.3.4".to_string(),
+            server_port: 6789,
+            public_server_url: None,
+            ..Configuration::new_sample()
+        };
+
+        let joined_url = config
+            .get_local_server_url()
+            .unwrap()
+            .join("some/path")
+            .unwrap();
+        assert!(
+            joined_url.as_str().contains(SERVER_BASE_PATH),
+            "Joined URL `{joined_url}`, does not contain base path `{SERVER_BASE_PATH}`"
+        );
+    }
+
+    #[test]
+    fn joining_to_public_server_url_without_trailing_slash() {
+        let subpath_without_trailing_slash = "subpath_without_trailing_slash";
+        let config = Configuration {
+            public_server_url: Some(format!(
+                "https://example.com/{subpath_without_trailing_slash}"
+            )),
+            ..Configuration::new_sample()
+        };
+
+        let joined_url = config.get_server_url().unwrap().join("some/path").unwrap();
+        assert!(
+            joined_url.as_str().contains(subpath_without_trailing_slash),
+            "Joined URL `{joined_url}`, does not contain subpath `{subpath_without_trailing_slash}`"
+        );
+    }
 }