Auto sync mithril fix mainnet

.config/dictionaries/project.dic

@@ -111,6 +111,7 @@ libtest
 linkat
 lintfix
 localizable
+logcall
 lookaside
 maindbname
 mapref
@@ -224,8 +225,8 @@ upnp
 utimensat
 vitss
 vkey
-vkeywitness
 VKEYS
+vkeywitness
 voteplan
 voteplans
 wallclock

hermes/Cargo.toml

@@ -64,6 +64,7 @@ thiserror = "1.0.56"
 hex = "0.4.3"
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+tracing-log = "0.2.0"
 test-log = { version = "0.2.16", default-features = false, features = [
     "trace",
 ] }
@@ -77,9 +78,9 @@ ed25519-bip32 = "0.4.1"
 dashmap = "6.0.1"
 once_cell = "1.19.0"
 clap = "4.5.3"
-build-info = "0.0.37"
-build-info-build = "0.0.37"
-derive_more = "0.99.17"
+build-info = "0.0.38"
+build-info-build = "0.0.38"
+derive_more = "1.0.0"
 chrono = "0.4.38"
 chrono-tz = "0.9.0"
 saffron = "0.1.0"
@@ -153,7 +154,7 @@ ed25519-dalek = "2.1.1"
 x509-cert = "0.2.5"
 coset = "0.3.7"
 libipld = "0.16.0"
-libp2p = "0.53.2"
+libp2p = "0.54.0"
 rust-ipfs = "0.11.19"
 rustyline-async = "0.4.2"
 ouroboros = "0.18.4"
@@ -164,4 +165,5 @@ redb = {version = "2.1.1", features = ["cache_metrics"]}
 heed = {version = "0.20.3", features = ["mdb_idl_logn_16", "posix-sem"]}
 brotli = "6.0.0"
 c509-certificate = { git = "https://github.com/input-output-hk/catalyst-voices.git", package = "c509-certificate", branch = "fix/c509-cleanup"}
-num-traits = "0.2.19"
+num-traits = "0.2.19"
+logcall = "0.1.9"

hermes/Justfile

@@ -30,8 +30,20 @@ run-mithril-download-example: code_format
     RUST_LOG="error,follow_chains=debug,cardano_chain_follower=debug,mithril-client=debug" \
     ./target/release/examples/follow_chains --preprod
 
+run-mithril-download-example-preview: code_format
+    cargo build -r --package cardano-chain-follower --example follow_chains --features mimalloc
+    RUST_LOG="error,follow_chains=debug,cardano_chain_follower=debug,mithril-client=debug" \
+    ./target/release/examples/follow_chains --preview
+
+# Run long running developer test for mithril downloading.
+run-mithril-download-example-mainnet: code_format
+    cargo build -r --package cardano-chain-follower --example follow_chains --features mimalloc
+    RUST_LOG="error,follow_chains=debug,cardano_chain_follower=debug,mithril-client=debug" \
+    ./target/release/examples/follow_chains --mainnet
+
 # Run long running developer test for mithril downloading.
 debug-heap-mithril-download-example:
     cargo build --package cardano-chain-follower --example follow_chains
     RUST_LOG="error,follow_chains=debug,cardano_chain_follower=debug,mithril-client=debug" \
-    heaptrack ./target/debug/examples/follow_chains --preprod
+    heaptrack ./target/debug/examples/follow_chains --preprod
+

hermes/crates/cardano-chain-follower/Cargo.toml

@@ -50,6 +50,8 @@ x509-cert.workspace = true
 ed25519-dalek.workspace = true
 blake2b_simd.workspace = true
 num-traits.workspace = true
+logcall.workspace = true
+tracing-log.workspace = true
 
 [dev-dependencies]
 hex.workspace = true

hermes/crates/cardano-chain-follower/examples/follow_chains.rs

@@ -39,6 +39,10 @@ fn process_argument() -> (Vec<Network>, ArgMatches) {
                 .action(ArgAction::SetTrue),
             arg!(--"halt-on-error" "Stop the process when an error occurs without retrying.")
                 .action(ArgAction::SetTrue),
+            arg!(--"bad-cip36" "Dump Bad Cip36 registrations detected.")
+                .action(ArgAction::SetTrue),
+            arg!(--"largest-metadata" "Dump The largest transaction metadata we find (as we find it).")
+                .action(ArgAction::SetTrue),
         ])
         .get_matches();
 
@@ -57,11 +61,8 @@ fn process_argument() -> (Vec<Network>, ArgMatches) {
 }
 
 /// Start syncing a particular network
-async fn start_sync_for(network: &Network, matches: ArgMatches) -> Result<(), Box<dyn Error>> {
-    let halt_on_error = matches.get_flag("halt-on-error");
-
-    let mut cfg = ChainSyncConfig::default_for(*network);
-    cfg.mithril_cfg.halt_on_error = halt_on_error;
+async fn start_sync_for(network: &Network) -> Result<(), Box<dyn Error>> {
+    let cfg = ChainSyncConfig::default_for(*network);
 
     info!(chain = cfg.chain.to_string(), "Starting Sync");
 
@@ -85,6 +86,9 @@ async fn follow_for(network: Network, matches: ArgMatches) {
     let all_tip_blocks = matches.get_flag("all-tip-blocks");
     let all_live_blocks = matches.get_flag("all-live-blocks");
     let stop_at_tip = matches.get_flag("stop-at-tip");
+    let halt_on_error = matches.get_flag("halt-on-error");
+    let bad_cip36 = matches.get_flag("bad-cip36");
+    let largest_metadata = matches.get_flag("largest-metadata");
 
     let mut current_era = String::new();
     let mut last_update: Option<ChainUpdate> = None;
@@ -98,6 +102,8 @@ async fn follow_for(network: Network, matches: ArgMatches) {
 
     let mut last_metrics_time = Instant::now();
 
+    let mut biggest_aux_data: usize = 0;
+
     while let Some(chain_update) = follower.next().await {
         updates += 1;
 
@@ -176,34 +182,34 @@ async fn follow_for(network: Network, matches: ArgMatches) {
                 .data
                 .txn_metadata(tx_idx, Metadata::cip36::LABEL)
             {
-                if !last_update_shown {
-                    info!(
-                        chain = network.to_string(),
-                        "Chain Update {updates}:{}", chain_update
-                    );
-                    // We already showed the last update, no need to show it again.
-                    last_update_shown = true;
-                }
-
                 let raw_size = match chain_update
                     .data
                     .txn_raw_metadata(tx_idx, Metadata::cip36::LABEL)
                 {
-                    Some(raw) => format!("Raw Size: {}", raw.len()),
-                    None => "Error: Cip36 Raw Metadata is missing".to_string(),
+                    Some(raw) => raw.len(),
+                    None => 0,
                 };
 
-                #[allow(irrefutable_let_patterns)] // Won't always be irrefutable.
-                if let Metadata::DecodedMetadataValues::Cip36(cip36) = &decoded_metadata.value {
-                    if !cip36.signed {
-                        dump_raw_aux_data = true;
-                    }
+                if largest_metadata && raw_size > biggest_aux_data {
+                    biggest_aux_data = raw_size;
+                    dump_raw_aux_data = true;
                 }
 
-                info!(
-                    chain = network.to_string(),
-                    "Cip36 {tx_idx}:{:?} - {raw_size}", decoded_metadata
-                );
+                if bad_cip36 {
+                    #[allow(irrefutable_let_patterns)] // Won't always be irrefutable.
+                    if let Metadata::DecodedMetadataValues::Cip36(cip36) = &decoded_metadata.value {
+                        if !cip36.signed {
+                            dump_raw_aux_data = true;
+                        }
+                        if !decoded_metadata.report.is_empty() {
+                            info!(
+                                chain = network.to_string(),
+                                "Cip36 {tx_idx}:{:?} - {raw_size}", decoded_metadata
+                            );
+                            dump_raw_aux_data = true;
+                        }
+                    }
+                }
             }
         }
 
@@ -240,6 +246,16 @@ async fn follow_for(network: Network, matches: ArgMatches) {
             let stats = Statistics::new(network);
 
             info!("Json Metrics:  {}", stats.as_json(true));
+
+            if halt_on_error
+                && (stats.mithril.download_or_validation_failed > 0
+                    || stats.mithril.failed_to_get_tip > 0
+                    || stats.mithril.tip_did_not_advance > 0
+                    || stats.mithril.tip_failed_to_send_to_updater > 0
+                    || stats.mithril.failed_to_activate_new_snapshot > 0)
+            {
+                break;
+            }
         }
     }
 
@@ -282,7 +298,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
     // First we need to actually start the underlying sync tasks for each blockchain.
     for network in &networks {
-        start_sync_for(network, matches.clone()).await?;
+        start_sync_for(network).await?;
     }
 
     // Make a follower for the network.

hermes/crates/cardano-chain-follower/src/chain_sync.rs

@@ -418,11 +418,6 @@ async fn live_sync_backfill_and_purge(
     // Wait for first Mithril Update advice, which triggers a BACKFILL of the Live Data.
     let Some(update) = rx.recv().await else {
         error!("Mithril Sync Failed, can not continue chain sync either.");
-
-        if cfg.mithril_cfg.halt_on_error {
-            std::process::exit(1);
-        }
-
         return;
     };
 

hermes/crates/cardano-chain-follower/src/follow.rs

@@ -98,7 +98,9 @@ impl ChainFollower {
 
             if let Some(follower) = self.mithril_follower.as_mut() {
                 if let Some(next) = follower.next().await {
+                    // debug!("Pre Previous update 3 : {:?}", self.previous);
                     self.previous = self.current.clone();
+                    // debug!("Post Previous update 3 : {:?}", self.previous);
                     self.current = next.point();
                     self.fork = 0; // Mithril Immutable data is always Fork 0.
                     let update = ChainUpdate::new(chain_update::Kind::Block, false, next);
@@ -117,7 +119,11 @@ impl ChainFollower {
                     ChainUpdate::new(chain_update::Kind::ImmutableBlockRollForward, false, block);
                 return Some(update);
             }
-            error!("Mithril Tip Block is not in snapshot. Should not happen.");
+            error!(
+                tip = ?self.mithril_tip,
+                current = ?current_mithril_tip,
+                "Mithril Tip Block is not in snapshot. Should not happen."
+            );
         }
 
         None
@@ -181,7 +187,9 @@ impl ChainFollower {
             if update_type == chain_update::Kind::Rollback {
                 rollback(self.chain, stats::RollbackType::Follower, rollback_depth);
             }
+            // debug!("Pre Previous update 4 : {:?}", self.previous);
             self.previous = self.current.clone();
+            // debug!("Post Previous update 4 : {:?}", self.previous);
             self.current = next_block.point().clone();
             self.fork = next_block.fork();
 

hermes/crates/cardano-chain-follower/src/metadata/cip36.rs

@@ -164,7 +164,7 @@ impl Cip36 {
                             )
                             .is_none()
                         {
-                            debug!("decoded 4: {decoded_metadata:?} : {validation_report:?} : {raw_cip36:02x?}");
+                            // debug!("decoded 4: {decoded_metadata:?} : {validation_report:?} : {raw_cip36:02x?}");
                             return;
                         }
                     },
@@ -177,7 +177,8 @@ impl Cip36 {
                             )
                             .is_none()
                         {
-                            debug!("decoded 5: {decoded_metadata:?} : {validation_report:?} : {raw_cip36:02x?}");
+                            // debug!("decoded 5: {decoded_metadata:?} : {validation_report:?} :
+                            // {raw_cip36:02x?}");
                             return;
                         }
                     },
@@ -722,18 +723,16 @@ impl Cip36 {
         }
 
         let sig: ed25519_dalek::Signature = match decoder.bytes() {
-            Ok(sig) => {
-                match ed25519_dalek::Signature::from_slice(sig) {
-                    Ok(sig) => sig,
-                    Err(err) => {
-                        self.decoding_failed(
-                            format!("CIP36 Signature Decoding failed: {err}",).as_str(),
-                            validation_report,
-                            decoded_metadata,
-                        );
-                        return None;
-                    },
-                }
+            Ok(sig) => match ed25519_dalek::Signature::from_slice(sig) {
+                Ok(sig) => sig,
+                Err(err) => {
+                    self.decoding_failed(
+                        format!("CIP36 Signature Decoding failed: {err}",).as_str(),
+                        validation_report,
+                        decoded_metadata,
+                    );
+                    return None;
+                },
             },
             Err(error) => {
                 self.decoding_failed(

hermes/crates/cardano-chain-follower/src/metadata/mod.rs

@@ -114,27 +114,46 @@ impl DecodedTransaction {
             decoded: SkipMap::new(),
         };
 
-        let transactions = block.txs();
-        let slot = block.slot();
-
-        if let Some(_metadata) = block.as_byron() {
-            // Nothing to do here.
-        } else if let Some(alonzo_block) = block.as_alonzo() {
-            for (txn_idx, metadata) in alonzo_block.auxiliary_data_set.iter() {
-                decoded_aux_data.insert(chain, slot, *txn_idx, metadata.raw_cbor(), &transactions);
-            }
-        } else if let Some(babbage_block) = block.as_babbage() {
-            for (txn_idx, metadata) in babbage_block.auxiliary_data_set.iter() {
-                decoded_aux_data.insert(chain, slot, *txn_idx, metadata.raw_cbor(), &transactions);
-            }
-        } else if let Some(conway_block) = block.as_conway() {
-            for (txn_idx, metadata) in conway_block.auxiliary_data_set.iter() {
-                decoded_aux_data.insert(chain, slot, *txn_idx, metadata.raw_cbor(), &transactions);
-            }
-        } else {
-            error!("Undecodable metadata, unknown Era");
-        };
-
+        if block.has_aux_data() {
+            let transactions = block.txs();
+            let slot = block.slot();
+
+            if let Some(_metadata) = block.as_byron() {
+                // Nothing to do here.
+            } else if let Some(alonzo_block) = block.as_alonzo() {
+                for (txn_idx, metadata) in alonzo_block.auxiliary_data_set.iter() {
+                    decoded_aux_data.insert(
+                        chain,
+                        slot,
+                        *txn_idx,
+                        metadata.raw_cbor(),
+                        &transactions,
+                    );
+                }
+            } else if let Some(babbage_block) = block.as_babbage() {
+                for (txn_idx, metadata) in babbage_block.auxiliary_data_set.iter() {
+                    decoded_aux_data.insert(
+                        chain,
+                        slot,
+                        *txn_idx,
+                        metadata.raw_cbor(),
+                        &transactions,
+                    );
+                }
+            } else if let Some(conway_block) = block.as_conway() {
+                for (txn_idx, metadata) in conway_block.auxiliary_data_set.iter() {
+                    decoded_aux_data.insert(
+                        chain,
+                        slot,
+                        *txn_idx,
+                        metadata.raw_cbor(),
+                        &transactions,
+                    );
+                }
+            } else {
+                error!("Undecodable metadata, unknown Era");
+            };
+        }
         decoded_aux_data
     }