From 22ec17caaa7f39180cd73bea937b789fff10aa5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:29:02 +0100 Subject: [PATCH 1/9] Fix activate immediately --- nano/lib/stats_enums.hpp | 1 + nano/node/active_elections.cpp | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 429741d2f3..19906ceb98 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -458,6 +458,7 @@ enum class detail transition_priority, transition_priority_failed, election_cleanup, + activate_immediately, // active_elections started, diff --git a/nano/node/active_elections.cpp b/nano/node/active_elections.cpp index b6dc0bd648..56d3fd1148 100644 --- a/nano/node/active_elections.cpp +++ b/nano/node/active_elections.cpp @@ -419,11 +419,16 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr< count_by_behavior[result.election->behavior ()]++; // Skip passive phase for blocks without cached votes to avoid bootstrap delays - bool active_immediately = false; - if (node.vote_cache.contains (hash)) + bool activate_immediately = false; + if (!node.vote_cache.contains (hash)) { + activate_immediately = true; + } + + if (activate_immediately) + { + node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::activate_immediately); result.election->transition_active (); - active_immediately = true; } node.stats.inc (nano::stat::type::active_elections, nano::stat::detail::started); @@ -436,7 +441,7 @@ nano::election_insertion_result nano::active_elections::insert (std::shared_ptr< node.logger.debug (nano::log::type::active_elections, "Started new election for block: {} (behavior: {}, active immediately: {})", hash.to_string (), to_string (election_behavior_a), - active_immediately); + activate_immediately); } else { From f7a3e52907988fbfc23689a67b6fa01be60db82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:41:05 +0100 Subject: [PATCH 2/9] Log frontiers --- nano/node/bootstrap/bootstrap_service.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nano/node/bootstrap/bootstrap_service.cpp b/nano/node/bootstrap/bootstrap_service.cpp index 8f60f9924f..88cd06e7e0 100644 --- a/nano/node/bootstrap/bootstrap_service.cpp +++ b/nano/node/bootstrap/bootstrap_service.cpp @@ -1047,6 +1047,8 @@ void nano::bootstrap_service::process_frontiers (std::deque guard{ mutex }; for (auto const & account : result) From fa912e97715ac413fc23580d24eb7918305c689a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:44:51 +0100 Subject: [PATCH 3/9] Log block processor suspicious activity --- nano/node/block_processor.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nano/node/block_processor.cpp b/nano/node/block_processor.cpp index dd72521c8c..a4f5858310 100644 --- a/nano/node/block_processor.cpp +++ b/nano/node/block_processor.cpp @@ -392,41 +392,51 @@ nano::block_status nano::block_processor::process_one (secure::write_transaction stats.inc (nano::stat::type::ledger, nano::stat::detail::old); break; } + // These are unexpected and indicate erroneous/malicious behavior, log debug info to highlight the issue case nano::block_status::bad_signature: { + logger.debug (nano::log::type::block_processor, "Block signature is invalid: {}", hash); break; } case nano::block_status::negative_spend: { + logger.debug (nano::log::type::block_processor, "Block spends negative amount: {}", hash); break; } case nano::block_status::unreceivable: { + logger.debug (nano::log::type::block_processor, "Block is unreceivable: {}", hash); break; } case nano::block_status::fork: { stats.inc (nano::stat::type::ledger, nano::stat::detail::fork); + logger.debug (nano::log::type::block_processor, "Block is a fork: {}", hash); break; } case nano::block_status::opened_burn_account: { + logger.debug (nano::log::type::block_processor, "Block opens burn account: {}", hash); break; } case nano::block_status::balance_mismatch: { + logger.debug (nano::log::type::block_processor, "Block balance mismatch: {}", hash); break; } case nano::block_status::representative_mismatch: { + logger.debug (nano::log::type::block_processor, "Block representative mismatch: {}", hash); break; } case nano::block_status::block_position: { + logger.debug (nano::log::type::block_processor, "Block is in incorrect position: {}", hash); break; } case nano::block_status::insufficient_work: { + logger.debug (nano::log::type::block_processor, "Block has insufficient work: {}", hash); break; } } From 38edde0ffbc2489e320fc6ad2759075a477339b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 9 Jan 2025 19:03:42 +0100 Subject: [PATCH 4/9] Fix optimistic request --- nano/node/bootstrap/bootstrap_service.cpp | 45 ++++++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/nano/node/bootstrap/bootstrap_service.cpp b/nano/node/bootstrap/bootstrap_service.cpp index 88cd06e7e0..b76281a399 100644 --- a/nano/node/bootstrap/bootstrap_service.cpp +++ b/nano/node/bootstrap/bootstrap_service.cpp @@ -515,33 +515,58 @@ bool nano::bootstrap_service::request (nano::account account, size_t count, std: // Check if the account picked has blocks, if it does, start the pull from the highest block if (auto info = ledger.store.account.get (transaction, account)) { - tag.type = query_type::blocks_by_hash; - // Probabilistically choose between requesting blocks from account frontier or confirmed frontier // Optimistic requests start from the (possibly unconfirmed) account frontier and are vulnerable to bootstrap poisoning // Safe requests start from the confirmed frontier and given enough time will eventually resolve forks bool optimistic_reuest = rng.random (100) < config.optimistic_request_percentage; - if (!optimistic_reuest) + + if (optimistic_reuest) // Optimistic request case { + stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::optimistic); + + tag.type = query_type::blocks_by_hash; + tag.start = info->head; + tag.hash = info->head; + + logger.debug (nano::log::type::bootstrap, "Requesting blocks for {} starting from account frontier: {} (optimistic: {})", + account.to_account (), // TODO: Lazy eval + tag.start.to_string (), // TODO: Lazy eval + optimistic_reuest); + } + else // Pessimistic (safe) request case + { + stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::safe); + if (auto conf_info = ledger.store.confirmation_height.get (transaction, account)) { - stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::safe); + tag.type = query_type::blocks_by_hash; tag.start = conf_info->frontier; tag.hash = conf_info->height; + + logger.debug (nano::log::type::bootstrap, "Requesting blocks for {} starting from confirmation frontier: {} (optimistic: {})", + account.to_account (), // TODO: Lazy eval + tag.start.to_string (), // TODO: Lazy eval + optimistic_reuest); + } + else + { + tag.type = query_type::blocks_by_account; + tag.start = account; + + logger.debug (nano::log::type::bootstrap, "Requesting blocks for {} starting from account root (optimistic: {})", + account.to_account (), // TODO: Lazy eval + optimistic_reuest); } - } - if (tag.start.is_zero ()) - { - stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::optimistic); - tag.start = info->head; - tag.hash = info->head; } } else { stats.inc (nano::stat::type::bootstrap_request_blocks, nano::stat::detail::base); + tag.type = query_type::blocks_by_account; tag.start = account; + + logger.debug (nano::log::type::bootstrap, "Requesting blocks for {}", account.to_account ()); // TODO: Lazy eval } } From 24776147b2b4486cf1427035bf9c5a4a931cb7d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 9 Jan 2025 19:33:35 +0100 Subject: [PATCH 5/9] Log election activity --- nano/lib/stats_enums.hpp | 1 + nano/node/election.cpp | 29 +++++++++++++++++++++++++---- nano/node/vote_router.cpp | 34 ++++++++++++++++++++++++---------- nano/node/vote_router.hpp | 2 ++ 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 19906ceb98..879c8b983d 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -284,6 +284,7 @@ enum class detail broadcast_block_repeat, confirm_once, confirm_once_failed, + confirmation_request, // election types manual, diff --git a/nano/node/election.cpp b/nano/node/election.cpp index 6780acfb14..7a7a267bfc 100644 --- a/nano/node/election.cpp +++ b/nano/node/election.cpp @@ -168,12 +168,24 @@ std::chrono::milliseconds nano::election::confirm_req_time () const void nano::election::send_confirm_req (nano::confirmation_solicitor & solicitor_a) { + debug_assert (!mutex.try_lock ()); + if (confirm_req_time () < (std::chrono::steady_clock::now () - last_req)) { if (!solicitor_a.add (*this)) { last_req = std::chrono::steady_clock::now (); ++confirmation_request_count; + + node.stats.inc (nano::stat::type::election, nano::stat::detail::confirmation_request); + node.logger.debug (nano::log::type::election, "Sent confirmation request for root: {} (behavior: {}, state: {}, voters: {}, blocks: {}, duration: {}ms, confirmation requests: {})", + qualified_root.to_string (), + to_string (behavior_m), + to_string (state_m), + status.voter_count, + status.block_count, + duration ().count (), + confirmation_request_count.load ()); } } } @@ -196,9 +208,10 @@ bool nano::election::transition_priority () behavior_m = nano::election_behavior::priority; last_vote = std::chrono::steady_clock::time_point{}; // allow new outgoing votes immediately - node.logger.debug (nano::log::type::election, "Transitioned election behavior to priority from {} for root: {}", + node.logger.debug (nano::log::type::election, "Transitioned election behavior to priority from {} for root: {} (duration: {}ms)", to_string (behavior_m), - qualified_root.to_string ()); + qualified_root.to_string (), + duration ().count ()); return true; } @@ -252,10 +265,18 @@ void nano::election::broadcast_block (nano::confirmation_solicitor & solicitor_a { if (!solicitor_a.broadcast (*this)) { - node.stats.inc (nano::stat::type::election, last_block_hash.is_zero () ? nano::stat::detail::broadcast_block_initial : nano::stat::detail::broadcast_block_repeat); - last_block = std::chrono::steady_clock::now (); last_block_hash = status.winner->hash (); + + node.stats.inc (nano::stat::type::election, last_block_hash.is_zero () ? nano::stat::detail::broadcast_block_initial : nano::stat::detail::broadcast_block_repeat); + node.logger.debug (nano::log::type::election, "Broadcasting current winner: {} for root: {} (behavior: {}, state: {}, voters: {}, blocks: {}, duration: {}ms)", + status.winner->hash ().to_string (), + qualified_root.to_string (), + to_string (behavior_m), + to_string (state_m), + status.voter_count, + status.block_count, + duration ().count ()); } } } diff --git a/nano/node/vote_router.cpp b/nano/node/vote_router.cpp index 75d641229b..95f350ee50 100644 --- a/nano/node/vote_router.cpp +++ b/nano/node/vote_router.cpp @@ -11,16 +11,6 @@ using namespace std::chrono_literals; -nano::stat::detail nano::to_stat_detail (nano::vote_code code) -{ - return nano::enum_util::cast (code); -} - -nano::stat::detail nano::to_stat_detail (nano::vote_source source) -{ - return nano::enum_util::cast (source); -} - nano::vote_router::vote_router (nano::vote_cache & vote_cache_a, nano::recently_confirmed_cache & recently_confirmed_a) : vote_cache{ vote_cache_a }, recently_confirmed{ recently_confirmed_a } @@ -201,3 +191,27 @@ nano::container_info nano::vote_router::container_info () const info.put ("elections", elections); return info; } + +/* + * + */ + +nano::stat::detail nano::to_stat_detail (nano::vote_code code) +{ + return nano::enum_util::cast (code); +} + +std::string_view nano::to_string (nano::vote_code code) +{ + return nano::enum_util::name (code); +} + +nano::stat::detail nano::to_stat_detail (nano::vote_source source) +{ + return nano::enum_util::cast (source); +} + +std::string_view nano::to_string (nano::vote_source source) +{ + return nano::enum_util::name (source); +} \ No newline at end of file diff --git a/nano/node/vote_router.hpp b/nano/node/vote_router.hpp index e74f06705d..6576196f60 100644 --- a/nano/node/vote_router.hpp +++ b/nano/node/vote_router.hpp @@ -21,6 +21,7 @@ enum class vote_code }; nano::stat::detail to_stat_detail (vote_code); +std::string_view to_string (vote_code); enum class vote_source { @@ -30,6 +31,7 @@ enum class vote_source }; nano::stat::detail to_stat_detail (vote_source); +std::string_view to_string (vote_source); // This class routes votes to their associated election // This class holds a weak_ptr as this container does not own the elections From 9d223ae71ac08a6aa178a6f1eb5c9b8a3583b68f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 9 Jan 2025 19:40:28 +0100 Subject: [PATCH 6/9] Log election votes --- nano/node/election.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nano/node/election.cpp b/nano/node/election.cpp index 7a7a267bfc..1f064b62e7 100644 --- a/nano/node/election.cpp +++ b/nano/node/election.cpp @@ -557,6 +557,14 @@ nano::vote_code nano::election::vote (nano::account const & rep, uint64_t timest nano::log::arg{ "vote_source", vote_source_a }, nano::log::arg{ "weight", weight }); + node.logger.debug (nano::log::type::election, "Vote received for: {} from: {} root: {} (final: {}, weight: {}, source: {})", + block_hash_a.to_string (), + rep.to_account (), + qualified_root.to_string (), + nano::vote::is_final_timestamp (timestamp_a), + weight.convert_to (), + to_string (vote_source_a)); + if (!confirmed_locked ()) { confirm_if_quorum (lock); From 418c8ec592463c75f2b6d584fe18f2568bc8d378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:22:31 +0100 Subject: [PATCH 7/9] Log request aggregator --- nano/lib/logging_enums.hpp | 1 + nano/node/node.cpp | 2 +- nano/node/request_aggregator.cpp | 20 +++++++++++++++++--- nano/node/request_aggregator.hpp | 5 +++-- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/nano/lib/logging_enums.hpp b/nano/lib/logging_enums.hpp index 64f520bc62..59d4cea5a1 100644 --- a/nano/lib/logging_enums.hpp +++ b/nano/lib/logging_enums.hpp @@ -86,6 +86,7 @@ enum class type monitor, confirming_set, bounded_backlog, + request_aggregator, // bootstrap bulk_pull_client, diff --git a/nano/node/node.cpp b/nano/node/node.cpp index b6838515fa..50d0ce297c 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -171,7 +171,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy final_generator{ *final_generator_impl }, scheduler_impl{ std::make_unique (config, *this, ledger, bucketing, block_processor, active, online_reps, vote_cache, confirming_set, stats, logger) }, scheduler{ *scheduler_impl }, - aggregator_impl{ std::make_unique (config.request_aggregator, *this, stats, generator, final_generator, history, ledger, wallets, vote_router) }, + aggregator_impl{ std::make_unique (config.request_aggregator, *this, generator, final_generator, history, ledger, wallets, vote_router) }, aggregator{ *aggregator_impl }, backlog_scan_impl{ std::make_unique (config.backlog_scan, ledger, stats) }, backlog_scan{ *backlog_scan_impl }, diff --git a/nano/node/request_aggregator.cpp b/nano/node/request_aggregator.cpp index 8a0c9224f4..d06a9b26ce 100644 --- a/nano/node/request_aggregator.cpp +++ b/nano/node/request_aggregator.cpp @@ -15,16 +15,17 @@ #include #include -nano::request_aggregator::request_aggregator (request_aggregator_config const & config_a, nano::node & node_a, nano::stats & stats_a, nano::vote_generator & generator_a, nano::vote_generator & final_generator_a, nano::local_vote_history & history_a, nano::ledger & ledger_a, nano::wallets & wallets_a, nano::vote_router & vote_router_a) : +nano::request_aggregator::request_aggregator (request_aggregator_config const & config_a, nano::node & node_a, nano::vote_generator & generator_a, nano::vote_generator & final_generator_a, nano::local_vote_history & history_a, nano::ledger & ledger_a, nano::wallets & wallets_a, nano::vote_router & vote_router_a) : config{ config_a }, network_constants{ node_a.network_params.network }, - stats (stats_a), local_votes (history_a), ledger (ledger_a), wallets (wallets_a), vote_router{ vote_router_a }, generator (generator_a), - final_generator (final_generator_a) + final_generator (final_generator_a), + stats (node_a.stats), + logger (node_a.logger) { queue.max_size_query = [this] (auto const & origin) { return config.max_queue; @@ -238,15 +239,28 @@ auto nano::request_aggregator::aggregate (nano::secure::transaction const & tran { to_generate_final.push_back (block); stats.inc (nano::stat::type::requests, nano::stat::detail::requests_final); + + logger.debug (nano::log::type::request_aggregator, "Replying with final vote for: {} to: {}", + block->hash ().to_string (), // TODO: Lazy eval + channel_a->to_string ()); // TODO: Lazy eval } else { stats.inc (nano::stat::type::requests, nano::stat::detail::requests_non_final); + + logger.debug (nano::log::type::request_aggregator, "Skipping reply with normal vote for: {} (requested by: {})", + block->hash ().to_string (), // TODO: Lazy eval + channel_a->to_string ()); // TODO: Lazy eval } } else { stats.inc (nano::stat::type::requests, nano::stat::detail::requests_unknown); + + logger.debug (nano::log::type::request_aggregator, "Cannot reply, block not found: {} with root: {} (requested by: {})", + hash.to_string (), // TODO: Lazy eval + root.to_string (), // TODO: Lazy eval + channel_a->to_string ()); // TODO: Lazy eval } } diff --git a/nano/node/request_aggregator.hpp b/nano/node/request_aggregator.hpp index c910720e13..1332875c2c 100644 --- a/nano/node/request_aggregator.hpp +++ b/nano/node/request_aggregator.hpp @@ -45,7 +45,7 @@ class request_aggregator_config final class request_aggregator final { public: - request_aggregator (request_aggregator_config const &, nano::node &, nano::stats &, nano::vote_generator &, nano::vote_generator &, nano::local_vote_history &, nano::ledger &, nano::wallets &, nano::vote_router &); + request_aggregator (request_aggregator_config const &, nano::node &, nano::vote_generator &, nano::vote_generator &, nano::local_vote_history &, nano::ledger &, nano::wallets &, nano::vote_router &); ~request_aggregator (); void start (); @@ -84,13 +84,14 @@ class request_aggregator final private: // Dependencies request_aggregator_config const & config; nano::network_constants const & network_constants; - nano::stats & stats; nano::local_vote_history & local_votes; nano::ledger & ledger; nano::wallets & wallets; nano::vote_router & vote_router; nano::vote_generator & generator; nano::vote_generator & final_generator; + nano::stats & stats; + nano::logger & logger; private: using value_type = std::pair>; From 9594e8b2dc240dfead15567062fce5f684725e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:28:51 +0100 Subject: [PATCH 8/9] Fix request aggregator reply for account open block --- nano/node/request_aggregator.cpp | 35 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/nano/node/request_aggregator.cpp b/nano/node/request_aggregator.cpp index d06a9b26ce..ab84cc60ce 100644 --- a/nano/node/request_aggregator.cpp +++ b/nano/node/request_aggregator.cpp @@ -202,21 +202,36 @@ auto nano::request_aggregator::aggregate (nano::secure::transaction const & tran { std::vector> to_generate; std::vector> to_generate_final; + for (auto const & [hash, root] : requests_a) { - // Ledger by hash - std::shared_ptr block = ledger.any.block_get (transaction, hash); + auto search_for_block = [&] () -> std::shared_ptr { + // Ledger by hash + if (auto block = ledger.any.block_get (transaction, hash)) + { + return block; + } - // Ledger by root - if (!block && !root.is_zero ()) - { - // Search for block root - if (auto successor = ledger.any.block_successor (transaction, root.as_block_hash ())) + // Ledger by root + if (!root.is_zero ()) { - block = ledger.any.block_get (transaction, successor.value ()); - release_assert (block); + // Search for successor of root + if (auto successor = ledger.any.block_successor (transaction, root.as_block_hash ())) + { + return ledger.any.block_get (transaction, successor.value ()); + } + + // If that fails treat root as account + if (auto info = ledger.any.account_get (transaction, root.as_account ())) + { + return ledger.any.block_get (transaction, info->open_block); + } } - } + + return nullptr; + }; + + auto block = search_for_block (); auto should_generate_final_vote = [&] (auto const & block) { release_assert (block); From 725f3cbfae81d8819e5cad048a89c5af00f5d924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:29:36 +0100 Subject: [PATCH 9/9] Fix `node.bootstrap_fork_open` test --- nano/core_test/node.cpp | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 697210c217..535a533ff3 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -1385,6 +1385,12 @@ TEST (node, bootstrap_fork_open) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); + node_config.bootstrap.account_sets.cooldown = 100ms; // Reduce cooldown to speed up fork resolution + // Disable automatic election activation + node_config.backlog_scan.enable = false; + node_config.priority_scheduler.enable = false; + node_config.hinted_scheduler.enable = false; + node_config.optimistic_scheduler.enable = false; auto node0 = system.add_node (node_config); node_config.peering_port = system.get_available_port (); auto node1 = system.add_node (node_config); @@ -1411,26 +1417,30 @@ TEST (node, bootstrap_fork_open) .sign (key0.prv, key0.pub) .work (*system.work.generate (key0.pub)) .build (); + // Both know about send0 ASSERT_EQ (nano::block_status::progress, node0->process (send0)); ASSERT_EQ (nano::block_status::progress, node1->process (send0)); + // Confirm send0 to allow starting and voting on the following blocks - for (auto node : system.nodes) - { - node->start_election (node->block (node->latest (nano::dev::genesis_key.pub))); - ASSERT_TIMELY (1s, node->active.election (send0->qualified_root ())); - auto election = node->active.election (send0->qualified_root ()); - ASSERT_NE (nullptr, election); - election->force_confirm (); - ASSERT_TIMELY (2s, node->active.empty ()); - } - ASSERT_TIMELY (3s, node0->block_confirmed (send0->hash ())); + nano::test::confirm (*node0, { send0 }); + nano::test::confirm (*node1, { send0 }); + ASSERT_TIMELY (5s, node0->block_confirmed (send0->hash ())); + ASSERT_TIMELY (5s, node1->block_confirmed (send0->hash ())); + // They disagree about open0/open1 ASSERT_EQ (nano::block_status::progress, node0->process (open0)); + node0->confirming_set.add (open0->hash ()); + ASSERT_TIMELY (5s, node0->block_confirmed (open0->hash ())); + ASSERT_EQ (nano::block_status::progress, node1->process (open1)); + ASSERT_TRUE (node1->block_or_pruned_exists (open1->hash ())); + node1->start_election (open1); // Start election for open block which is necessary to resolve the fork + ASSERT_TIMELY (5s, node1->active.active (*open1)); + + // Allow node0 to vote on its fork system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - ASSERT_FALSE (node1->block_or_pruned_exists (open0->hash ())); - ASSERT_TIMELY (1s, node1->active.empty ()); + ASSERT_TIMELY (10s, !node1->block_or_pruned_exists (open1->hash ()) && node1->block_or_pruned_exists (open0->hash ())); }