diff --git a/ydb/core/blobstorage/base/blobstorage_events.cpp b/ydb/core/blobstorage/base/blobstorage_events.cpp index 1e3647ea232a..81f2341377fa 100644 --- a/ydb/core/blobstorage/base/blobstorage_events.cpp +++ b/ydb/core/blobstorage/base/blobstorage_events.cpp @@ -4,11 +4,12 @@ namespace NKikimr { TEvNodeWardenStorageConfig::TEvNodeWardenStorageConfig(const NKikimrBlobStorage::TStorageConfig& config, - const NKikimrBlobStorage::TStorageConfig *proposedConfig) + const NKikimrBlobStorage::TStorageConfig *proposedConfig, bool selfManagementEnabled) : Config(std::make_unique(config)) , ProposedConfig(proposedConfig ? std::make_unique(*proposedConfig) : nullptr) + , SelfManagementEnabled(selfManagementEnabled) {} TEvNodeWardenStorageConfig::~TEvNodeWardenStorageConfig() diff --git a/ydb/core/blobstorage/base/blobstorage_events.h b/ydb/core/blobstorage/base/blobstorage_events.h index c7da3105d494..24efbd16a10f 100644 --- a/ydb/core/blobstorage/base/blobstorage_events.h +++ b/ydb/core/blobstorage/base/blobstorage_events.h @@ -575,9 +575,10 @@ namespace NKikimr { { std::unique_ptr Config; std::unique_ptr ProposedConfig; + bool SelfManagementEnabled; TEvNodeWardenStorageConfig(const NKikimrBlobStorage::TStorageConfig& config, - const NKikimrBlobStorage::TStorageConfig *proposedConfig); + const NKikimrBlobStorage::TStorageConfig *proposedConfig, bool selfManagementEnabled); ~TEvNodeWardenStorageConfig(); }; diff --git a/ydb/core/blobstorage/nodewarden/distconf.cpp b/ydb/core/blobstorage/nodewarden/distconf.cpp index 13bcf42470e4..1e7ee8fcf88d 100644 --- a/ydb/core/blobstorage/nodewarden/distconf.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf.cpp @@ -34,10 +34,13 @@ namespace NKikimr::NStorage { // generate initial drive set and query stored configuration if (IsSelfStatic) { - EnumerateConfigDrives(InitialConfig, SelfId().NodeId(), [&](const auto& /*node*/, const auto& drive) { - DrivesToRead.push_back(drive.GetPath()); - }); - std::sort(DrivesToRead.begin(), DrivesToRead.end()); + if (BaseConfig.GetSelfManagementConfig().GetEnabled()) { + // read this only if it is possibly enabled + EnumerateConfigDrives(InitialConfig, SelfId().NodeId(), [&](const auto& /*node*/, const auto& drive) { + DrivesToRead.push_back(drive.GetPath()); + }); + std::sort(DrivesToRead.begin(), DrivesToRead.end()); + } ReadConfig(); } else { StorageConfigLoaded = true; @@ -91,6 +94,10 @@ namespace NKikimr::NStorage { } } + SelfManagementEnabled = (!IsSelfStatic || BaseConfig.GetSelfManagementConfig().GetEnabled()) && + config.GetSelfManagementConfig().GetEnabled() && + config.GetGeneration(); + StorageConfig.emplace(config); if (ProposedStorageConfig && ProposedStorageConfig->GetGeneration() <= StorageConfig->GetGeneration()) { ProposedStorageConfig.reset(); @@ -292,14 +299,13 @@ namespace NKikimr::NStorage { void TDistributedConfigKeeper::ReportStorageConfigToNodeWarden(ui64 cookie) { Y_ABORT_UNLESS(StorageConfig); const TActorId wardenId = MakeBlobStorageNodeWardenID(SelfId().NodeId()); - const bool distconfEnabled = StorageConfig->GetSelfManagementConfig().GetEnabled(); - const NKikimrBlobStorage::TStorageConfig *config = distconfEnabled + const NKikimrBlobStorage::TStorageConfig *config = SelfManagementEnabled ? &StorageConfig.value() : &BaseConfig; - const NKikimrBlobStorage::TStorageConfig *proposedConfig = ProposedStorageConfig && distconfEnabled + const NKikimrBlobStorage::TStorageConfig *proposedConfig = ProposedStorageConfig && SelfManagementEnabled ? &ProposedStorageConfig.value() : nullptr; - auto ev = std::make_unique(*config, proposedConfig); + auto ev = std::make_unique(*config, proposedConfig, SelfManagementEnabled); Send(wardenId, ev.release(), 0, cookie); } diff --git a/ydb/core/blobstorage/nodewarden/distconf.h b/ydb/core/blobstorage/nodewarden/distconf.h index 2ce2c4633e47..3fd49118fecd 100644 --- a/ydb/core/blobstorage/nodewarden/distconf.h +++ b/ydb/core/blobstorage/nodewarden/distconf.h @@ -174,6 +174,7 @@ namespace NKikimr::NStorage { const bool IsSelfStatic = false; TIntrusivePtr Cfg; + bool SelfManagementEnabled = false; // currently active storage config std::optional StorageConfig; diff --git a/ydb/core/blobstorage/nodewarden/distconf_console.cpp b/ydb/core/blobstorage/nodewarden/distconf_console.cpp index 63e98dc7cab1..6c5f2ab959bf 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_console.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_console.cpp @@ -12,10 +12,8 @@ namespace NKikimr::NStorage { return; // this is not the root node } else if (enablingDistconf) { // NO RETURN HERE -> right now we are enabling distconf, so we can skip rest of the checks - } else if (!StorageConfig || !StorageConfig->GetSelfManagementConfig().GetEnabled()) { - return; // no self-management config enabled - } else if (!StorageConfig->HasStateStorageConfig()) { - return; // no way to find Console too + } else if (!SelfManagementEnabled || !StorageConfig->HasStateStorageConfig()) { + return; // no self-management config enabled or no way to find Console (no statestorage configured yet) } STLOG(PRI_DEBUG, BS_NODE, NWDC66, "ConnectToConsole: creating pipe to the Console"); diff --git a/ydb/core/blobstorage/nodewarden/distconf_dynamic.cpp b/ydb/core/blobstorage/nodewarden/distconf_dynamic.cpp index dcfcfcfa481c..1f6a1ec27ce8 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_dynamic.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_dynamic.cpp @@ -109,7 +109,7 @@ namespace NKikimr::NStorage { ev->Record.SetNoQuorum(true); } else if (!StorageConfig) { // no storage configuration -- no nothing - } else if (auto *target = record.MutableConfig(); StorageConfig->GetSelfManagementConfig().GetEnabled()) { + } else if (auto *target = record.MutableConfig(); SelfManagementEnabled) { target->CopyFrom(*StorageConfig); } else { target->CopyFrom(BaseConfig); diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp index bcab5eeb0a5c..97265d91d53d 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke.cpp @@ -311,10 +311,10 @@ namespace NKikimr::NStorage { const auto& record = Event->Get()->Record; const auto& cmd = record.GetReassignGroupDisk(); - if (Scepter.expired()) { - return FinishWithError(TResult::ERROR, "scepter lost during query execution"); - } else if (!RunCommonChecks()) { + if (!RunCommonChecks()) { return; + } else if (!Self->SelfManagementEnabled) { + return FinishWithError(TResult::ERROR, "self-management is not enabled"); } STLOG(PRI_DEBUG, BS_NODE, NWDC75, "ReassignGroupDiskExecute", (SelfId, SelfId())); @@ -383,9 +383,6 @@ namespace NKikimr::NStorage { } const auto& ss = bsConfig.GetServiceSet(); - if (!config.GetSelfManagementConfig().GetEnabled()) { - return FinishWithError(TResult::ERROR, "self-management is not enabled"); - } const auto& smConfig = config.GetSelfManagementConfig(); THashMap replacedDisks; @@ -656,7 +653,7 @@ namespace NKikimr::NStorage { void ReplaceStorageConfig(const TQuery::TReplaceStorageConfig& request) { if (!RunCommonChecks()) { return; - } else if (!Self->ConfigCommittedToConsole && Self->StorageConfig->GetSelfManagementConfig().GetEnabled()) { + } else if (!Self->ConfigCommittedToConsole && Self->SelfManagementEnabled) { return FinishWithError(TResult::ERROR, "previous config has not been committed to Console yet"); } @@ -711,7 +708,8 @@ namespace NKikimr::NStorage { } // whether we are enabling distconf right now - const bool enablingDistconf = Self->StorageConfig->GetSelfManagementConfig().GetEnabled() < + const bool enablingDistconf = Self->BaseConfig.GetSelfManagementConfig().GetEnabled() && + !Self->SelfManagementEnabled && config.GetSelfManagementConfig().GetEnabled(); if (!Self->EnqueueConsoleConfigValidation(SelfId(), enablingDistconf, NewYaml)) { @@ -863,6 +861,8 @@ namespace NKikimr::NStorage { FinishWithError(TResult::ERROR, "something going on with default FSM"); } else if (auto error = ValidateConfig(*Self->StorageConfig)) { FinishWithError(TResult::ERROR, TStringBuilder() << "current config validation failed: " << *error); + } else if (Scepter.expired()) { + FinishWithError(TResult::ERROR, "scepter lost during query execution"); } else { return true; } diff --git a/ydb/core/blobstorage/nodewarden/distconf_mon.cpp b/ydb/core/blobstorage/nodewarden/distconf_mon.cpp index 6891269450cb..1a93f4453826 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_mon.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_mon.cpp @@ -153,6 +153,15 @@ namespace NKikimr::NStorage { } } + DIV_CLASS("panel panel-info") { + DIV_CLASS("panel-heading") { + out << "Main operational parameters"; + } + DIV_CLASS("panel-body") { + out << "Self-management enabled: " << (SelfManagementEnabled ? "yes" : "no") << "
"; + } + } + auto outputConfig = [&](const char *name, auto *config) { DIV_CLASS("panel panel-info") { DIV_CLASS("panel-heading") { diff --git a/ydb/core/blobstorage/nodewarden/distconf_persistent_storage.cpp b/ydb/core/blobstorage/nodewarden/distconf_persistent_storage.cpp index cbf6ad22da68..ec86d2952e1c 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_persistent_storage.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_persistent_storage.cpp @@ -293,12 +293,15 @@ namespace NKikimr::NStorage { // generate new list of drives to acquire std::vector drivesToRead; - EnumerateConfigDrives(InitialConfig, SelfId().NodeId(), [&](const auto& /*node*/, const auto& drive) { - drivesToRead.push_back(drive.GetPath()); - }); - std::sort(drivesToRead.begin(), drivesToRead.end()); + if (BaseConfig.GetSelfManagementConfig().GetEnabled()) { + EnumerateConfigDrives(InitialConfig, SelfId().NodeId(), [&](const auto& /*node*/, const auto& drive) { + drivesToRead.push_back(drive.GetPath()); + }); + std::sort(drivesToRead.begin(), drivesToRead.end()); + } if (DrivesToRead != drivesToRead) { // re-read configuration as it may cover additional drives + DrivesToRead = std::move(drivesToRead); ReadConfig(); } else { ApplyStorageConfig(InitialConfig); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h index 092479760ba7..120b5c711380 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h @@ -586,6 +586,7 @@ namespace NKikimr::NStorage { void ForwardToDistributedConfigKeeper(STATEFN_SIG); NKikimrBlobStorage::TStorageConfig StorageConfig; + bool SelfManagementEnabled = false; THashSet StorageConfigSubscribers; void Handle(TEvNodeWardenQueryStorageConfig::TPtr ev); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp b/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp index f249511cc87c..fe6b1c81dab1 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_mon.cpp @@ -105,6 +105,7 @@ void TNodeWarden::RenderWholePage(IOutputStream& out) { TAG(TH3) { out << "StorageConfig"; } DIV() { + out << "

Self-management enabled: " << (SelfManagementEnabled ? "yes" : "no") << "

"; TString s; NProtoBuf::TextFormat::PrintToString(StorageConfig, &s); out << "
" << s << "
"; diff --git a/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp b/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp index 61d60fea19fd..b64eed6e99d5 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_resource.cpp @@ -75,7 +75,7 @@ void TNodeWarden::ApplyServiceSet(const NKikimrBlobStorage::TNodeWardenServiceSe } void TNodeWarden::Handle(TEvNodeWardenQueryStorageConfig::TPtr ev) { - Send(ev->Sender, new TEvNodeWardenStorageConfig(StorageConfig, nullptr)); + Send(ev->Sender, new TEvNodeWardenStorageConfig(StorageConfig, nullptr, SelfManagementEnabled)); if (ev->Get()->Subscribe) { StorageConfigSubscribers.insert(ev->Sender); } @@ -83,6 +83,8 @@ void TNodeWarden::Handle(TEvNodeWardenQueryStorageConfig::TPtr ev) { void TNodeWarden::Handle(TEvNodeWardenStorageConfig::TPtr ev) { ev->Get()->Config->Swap(&StorageConfig); + SelfManagementEnabled = ev->Get()->SelfManagementEnabled; + if (StorageConfig.HasBlobStorageConfig()) { if (const auto& bsConfig = StorageConfig.GetBlobStorageConfig(); bsConfig.HasServiceSet()) { const NKikimrBlobStorage::TNodeWardenServiceSet *proposed = nullptr; @@ -98,15 +100,18 @@ void TNodeWarden::Handle(TEvNodeWardenStorageConfig::TPtr ev) { ApplyStorageConfig(bsConfig.GetServiceSet(), proposed); } } + if (StorageConfig.HasStateStorageConfig() && StorageConfig.HasStateStorageBoardConfig() && StorageConfig.HasSchemeBoardConfig()) { ApplyStateStorageConfig(ev->Get()->ProposedConfig.get()); } else { Y_ABORT_UNLESS(!StorageConfig.HasStateStorageConfig() && !StorageConfig.HasStateStorageBoardConfig() && !StorageConfig.HasSchemeBoardConfig()); } + for (const TActorId& subscriber : StorageConfigSubscribers) { - Send(subscriber, new TEvNodeWardenStorageConfig(StorageConfig, nullptr)); + Send(subscriber, new TEvNodeWardenStorageConfig(StorageConfig, nullptr, SelfManagementEnabled)); } + TActivationContext::Send(new IEventHandle(TEvBlobStorage::EvNodeWardenStorageConfigConfirm, 0, ev->Sender, SelfId(), nullptr, ev->Cookie)); } diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/node_warden_mock_bsc.cpp b/ydb/core/blobstorage/ut_blobstorage/lib/node_warden_mock_bsc.cpp index cf2b1e1386cd..0515916bdc31 100644 --- a/ydb/core/blobstorage/ut_blobstorage/lib/node_warden_mock_bsc.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/lib/node_warden_mock_bsc.cpp @@ -226,7 +226,7 @@ void TNodeWardenMockActor::Handle(TEvBlobStorage::TEvControllerNodeServiceSetUpd } void TNodeWardenMockActor::Handle(TEvNodeWardenQueryStorageConfig::TPtr ev) { - Send(ev->Sender, new TEvNodeWardenStorageConfig(NKikimrBlobStorage::TStorageConfig(), nullptr)); + Send(ev->Sender, new TEvNodeWardenStorageConfig(NKikimrBlobStorage::TStorageConfig(), nullptr, false)); } void TNodeWardenMockActor::HandleUnsubscribe(STATEFN_SIG) { diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp index 46f7be8a2af8..af77bca05388 100644 --- a/ydb/core/mind/bscontroller/bsc.cpp +++ b/ydb/core/mind/bscontroller/bsc.cpp @@ -97,6 +97,8 @@ NKikimrBlobStorage::TGroupStatus::E TBlobStorageController::DeriveStatus(const T } void TBlobStorageController::OnActivateExecutor(const TActorContext&) { + StartConsoleInteraction(); + // create stat processor StatProcessorActorId = Register(CreateStatProcessorActor()); @@ -124,6 +126,7 @@ void TBlobStorageController::OnActivateExecutor(const TActorContext&) { void TBlobStorageController::Handle(TEvNodeWardenStorageConfig::TPtr ev) { ev->Get()->Config->Swap(&StorageConfig); + SelfManagementEnabled = ev->Get()->SelfManagementEnabled; auto prevStaticPDisks = std::exchange(StaticPDisks, {}); auto prevStaticVSlots = std::exchange(StaticVSlots, {}); @@ -158,20 +161,21 @@ void TBlobStorageController::Handle(TEvNodeWardenStorageConfig::TPtr ev) { } } - if (StorageConfig.GetSelfManagementConfig().GetEnabled()) { + if (SelfManagementEnabled) { // assuming that in autoconfig mode HostRecords are managed by the distconf; we need to apply it here to // avoid race with box autoconfiguration and node list change HostRecords = std::make_shared(StorageConfig); if (SelfHealId) { Send(SelfHealId, new TEvPrivate::TEvUpdateHostRecords(HostRecords)); } + + ConsoleInteraction->Stop(); // distconf will handle the Console from now on } else { - StartConsoleInteraction(); - ConsoleInteraction->Start(); // start console interaction when working in non-distconf mode + ConsoleInteraction->Start(); // we control the Console now } if (!std::exchange(StorageConfigObtained, true)) { // this is the first time we get StorageConfig in this instance of BSC - if (HostRecords) { + if (SelfManagementEnabled) { OnHostRecordsInitiate(); } else { Send(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes(true)); @@ -192,9 +196,8 @@ void TBlobStorageController::Handle(TEvents::TEvUndelivered::TPtr ev) { } void TBlobStorageController::ApplyStorageConfig(bool ignoreDistconf) { - if (!StorageConfig.HasBlobStorageConfig() || // this would be strange - !ignoreDistconf && (!StorageConfig.GetSelfManagementConfig().GetEnabled() || - !StorageConfig.GetSelfManagementConfig().GetAutomaticBoxManagement())) { + if (!StorageConfig.HasBlobStorageConfig()) { + Y_DEBUG_ABORT("missing BlobStorageConfig with running BSC"); return; } const auto& bsConfig = StorageConfig.GetBlobStorageConfig(); @@ -202,6 +205,11 @@ void TBlobStorageController::ApplyStorageConfig(bool ignoreDistconf) { if (Boxes.size() > 1) { return; } + + if (!ignoreDistconf && (!SelfManagementEnabled || !StorageConfig.GetSelfManagementConfig().GetAutomaticBoxManagement())) { + return; // not expected to be managed by BSC + } + std::optional generation; if (!Boxes.empty()) { const auto& [boxId, box] = *Boxes.begin(); @@ -292,11 +300,10 @@ void TBlobStorageController::OnHostRecordsInitiate() { "BlobStorageControllerControls.EnableSelfHealWithDegraded"); } } + Y_ABORT_UNLESS(!SelfHealId); SelfHealId = Register(CreateSelfHealActor()); PushStaticGroupsToSelfHeal(); - if (StorageConfigObtained) { - Execute(CreateTxInitScheme()); - } + Execute(CreateTxInitScheme()); } void TBlobStorageController::IssueInitialGroupContent() { @@ -471,9 +478,9 @@ void TBlobStorageController::PassAway() { TActivationContext::Send(new IEventHandle(TEvents::TSystem::Unsubscribe, 0, MakeBlobStorageNodeWardenID(SelfId().NodeId()), SelfId(), nullptr, 0)); if (ConsoleInteraction) { - ConsoleInteraction->OnPassAway(); + ConsoleInteraction->Stop(); } - return TActor::PassAway(); + TActor::PassAway(); } TBlobStorageController::TBlobStorageController(const TActorId &tablet, TTabletStorageInfo *info) diff --git a/ydb/core/mind/bscontroller/console_interaction.cpp b/ydb/core/mind/bscontroller/console_interaction.cpp index 8b5f4fd55cd8..25f796bd7af3 100644 --- a/ydb/core/mind/bscontroller/console_interaction.cpp +++ b/ydb/core/mind/bscontroller/console_interaction.cpp @@ -35,6 +35,7 @@ namespace NKikimr::NBsController { proposeConfigEv->Record.SetConfigHash(configHash); proposeConfigEv->Record.SetConfigVersion(Self.ConfigVersion); NTabletPipe::SendData(Self.SelfId(), ConsolePipe, proposeConfigEv.Release()); + Working = true; } void TBlobStorageController::TConsoleInteraction::Handle(TEvTabletPipe::TEvClientConnected::TPtr& /*ev*/) { @@ -42,10 +43,12 @@ namespace NKikimr::NBsController { void TBlobStorageController::TConsoleInteraction::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& /*ev*/) { ConsolePipe = {}; - MakeGetBlock(); + if (Working) { + MakeGetBlock(); + } } - void TBlobStorageController::TConsoleInteraction::MakeCommitToConsole(TString& /* config */, ui32 /* configVersion */) { + void TBlobStorageController::TConsoleInteraction::MakeCommitToConsole(TString& /*config*/, ui32 /*configVersion*/) { auto ev = MakeHolder(); ev->Record.SetYAML(Self.YamlConfig); NTabletPipe::SendData(Self.SelfId(), ConsolePipe, ev.Release()); @@ -63,6 +66,9 @@ namespace NKikimr::NBsController { } void TBlobStorageController::TConsoleInteraction::Handle(TEvBlobStorage::TEvControllerProposeConfigResponse::TPtr &ev) { + if (!Working) { + return; + } STLOG(PRI_DEBUG, BS_CONTROLLER, BSC19, "Console proposed config response", (Response, ev->Get()->Record)); auto& record = ev->Get()->Record; auto status = record.GetStatus(); @@ -88,6 +94,9 @@ namespace NKikimr::NBsController { } void TBlobStorageController::TConsoleInteraction::OnConfigCommit(const TCommitConfigResult& result) { + if (!Working) { + return; + } if (result.Status == TCommitConfigResult::EStatus::ParseError || result.Status == TCommitConfigResult::EStatus::ValidationError) { auto response = std::make_unique(); @@ -105,13 +114,17 @@ namespace NKikimr::NBsController { } } - void TBlobStorageController::TConsoleInteraction::OnPassAway() { + void TBlobStorageController::TConsoleInteraction::Stop() { if (ConsolePipe) { NTabletPipe::CloseClient(Self.SelfId(), ConsolePipe); } + Working = false; } void TBlobStorageController::TConsoleInteraction::Handle(TEvBlobStorage::TEvControllerConsoleCommitResponse::TPtr &ev) { + if (!Working) { + return; + } STLOG(PRI_DEBUG, BS_CONTROLLER, BSC20, "Console commit config response", (Response, ev->Get()->Record)); auto& record = ev->Get()->Record; auto status = record.GetStatus(); @@ -139,7 +152,7 @@ namespace NKikimr::NBsController { STLOG(PRI_DEBUG, BS_CONTROLLER, BSC24, "Console replace config request", (Request, ev->Get()->Record)); auto& record = ev->Get()->Record; auto response = MakeHolder(); - if (!record.GetOverwriteFlag() && GRPCSenderId) { + if (!Working || (!record.GetOverwriteFlag() && GRPCSenderId)) { response->Record.SetStatus(NKikimrBlobStorage::TEvControllerReplaceConfigResponse::OngoingCommit); Self.Send(ev->Sender, response.Release()); return; @@ -232,6 +245,9 @@ namespace NKikimr::NBsController { } void TBlobStorageController::TConsoleInteraction::Handle(TEvBlobStorage::TEvGetBlockResult::TPtr& ev) { + if (!Working) { + return; + } auto* msg = ev->Get(); auto status = msg->Status; auto blockedGeneration = msg->BlockedGeneration; diff --git a/ydb/core/mind/bscontroller/console_interaction.h b/ydb/core/mind/bscontroller/console_interaction.h index c302289eb8c9..6455772b19e8 100644 --- a/ydb/core/mind/bscontroller/console_interaction.h +++ b/ydb/core/mind/bscontroller/console_interaction.h @@ -31,7 +31,7 @@ namespace NKikimr::NBsController { TConsoleInteraction(TBlobStorageController& controller); void Start(); void OnConfigCommit(const TCommitConfigResult& result); - void OnPassAway(); + void Stop(); bool ParseConfig(const TString& config, ui32& configVersion, NKikimrBlobStorage::TStorageConfig& storageConfig); TCommitConfigResult::EStatus CheckConfig(const TString& config, ui32& configVersion, bool skipBSCValidation, NKikimrBlobStorage::TStorageConfig& storageConfig); @@ -53,6 +53,7 @@ namespace NKikimr::NBsController { TBackoffTimer GetBlockBackoff{1, 1000}; ui32 BlockedGeneration = 0; bool NeedRetrySession = false; + bool Working = false; void MakeCommitToConsole(TString& config, ui32 configVersion); void MakeGetBlock(); diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h index 83a56adb5aee..0324cecae2d9 100644 --- a/ydb/core/mind/bscontroller/impl.h +++ b/ydb/core/mind/bscontroller/impl.h @@ -1539,6 +1539,7 @@ class TBlobStorageController : public TActor, public TTa bool DonorMode = false; TDuration ScrubPeriodicity; NKikimrBlobStorage::TStorageConfig StorageConfig; + bool SelfManagementEnabled = false; TString YamlConfig; ui32 ConfigVersion = 0; TBackoffTimer GetBlockBackoff{1, 1000}; @@ -1572,7 +1573,6 @@ class TBlobStorageController : public TActor, public TTa class TNodeWardenUpdateNotifier; class TConsoleInteraction; - std::unique_ptr ConsoleInteraction; struct TEvPrivate { @@ -2112,10 +2112,6 @@ class TBlobStorageController : public TActor, public TTa Loaded = true; ApplyStorageConfig(); - if (!ConsoleInteraction) { // maybe we are in distconf mode, no console interaction yet started - StartConsoleInteraction(); - } - for (const auto& [id, info] : GroupMap) { if (info->VirtualGroupState) { StartVirtualGroupSetupMachine(info.Get()); diff --git a/ydb/core/mind/bscontroller/self_heal.cpp b/ydb/core/mind/bscontroller/self_heal.cpp index bfb57b1a66f9..7618d1a93e03 100644 --- a/ydb/core/mind/bscontroller/self_heal.cpp +++ b/ydb/core/mind/bscontroller/self_heal.cpp @@ -955,8 +955,7 @@ namespace NKikimr::NBsController { } void TBlobStorageController::PushStaticGroupsToSelfHeal() { - if (!SelfHealId || !StorageConfigObtained || !StorageConfig.HasBlobStorageConfig() || - !StorageConfig.GetSelfManagementConfig().GetEnabled()) { + if (!SelfHealId || !StorageConfigObtained || !StorageConfig.HasBlobStorageConfig() || !SelfManagementEnabled) { return; } diff --git a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp index a8681553fe7f..4ca419ba1b03 100644 --- a/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp +++ b/ydb/core/mind/bscontroller/ut_bscontroller/main.cpp @@ -243,7 +243,7 @@ struct TEnvironmentSetup { {} void Handle(TEvNodeWardenQueryStorageConfig::TPtr ev) { - Send(ev->Sender, new TEvNodeWardenStorageConfig(NKikimrBlobStorage::TStorageConfig(), nullptr)); + Send(ev->Sender, new TEvNodeWardenStorageConfig(NKikimrBlobStorage::TStorageConfig(), nullptr, false)); } STATEFN(StateFunc) { diff --git a/ydb/core/mind/bscontroller/ut_selfheal/node_warden_mock.h b/ydb/core/mind/bscontroller/ut_selfheal/node_warden_mock.h index 5bbf78d1cb65..5697fb6be4d0 100644 --- a/ydb/core/mind/bscontroller/ut_selfheal/node_warden_mock.h +++ b/ydb/core/mind/bscontroller/ut_selfheal/node_warden_mock.h @@ -180,7 +180,7 @@ class TNodeWardenMock : public TActorBootstrapped { } void Handle(TEvNodeWardenQueryStorageConfig::TPtr ev) { - Send(ev->Sender, new TEvNodeWardenStorageConfig(NKikimrBlobStorage::TStorageConfig(), nullptr)); + Send(ev->Sender, new TEvNodeWardenStorageConfig(NKikimrBlobStorage::TStorageConfig(), nullptr, false)); } STRICT_STFUNC(StateFunc, { diff --git a/ydb/core/mind/dynamic_nameserver.cpp b/ydb/core/mind/dynamic_nameserver.cpp index c46b4ff3c2fb..a68fa312faab 100644 --- a/ydb/core/mind/dynamic_nameserver.cpp +++ b/ydb/core/mind/dynamic_nameserver.cpp @@ -146,22 +146,35 @@ void TDynamicNameserver::Bootstrap(const TActorContext &ctx) void TDynamicNameserver::Handle(TEvNodeWardenStorageConfig::TPtr ev) { Y_ABORT_UNLESS(ev->Get()->Config); const auto& config = *ev->Get()->Config; - if (config.GetSelfManagementConfig().GetEnabled()) { + std::unique_ptr consoleQuery; + + if (ev->Get()->SelfManagementEnabled) { // self-management through distconf is enabled and we are operating based on their tables, so apply them now - auto newStaticConfig = BuildNameserverTable(config); - if (StaticConfig->StaticNodeTable != newStaticConfig->StaticNodeTable) { - StaticConfig = std::move(newStaticConfig); - ListNodesCache->Invalidate(); - for (const auto& subscriber : StaticNodeChangeSubscribers) { - TActivationContext::Send(new IEventHandle(SelfId(), subscriber, new TEvInterconnect::TEvListNodes)); - } + ReplaceNameserverSetup(BuildNameserverTable(config)); + + // unsubscribe from console if we were operating without self-management before + if (std::exchange(SubscribedToConsole, false)) { + consoleQuery = std::make_unique(SelfId()); } - } else if (!SubscribedToConsole) { - Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()), new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest( + } else if (!std::exchange(SubscribedToConsole, true)) { + consoleQuery = std::make_unique( NKikimrConsole::TConfigItem::NameserviceConfigItem, SelfId() - )); - SubscribedToConsole = true; + ); + } + + if (consoleQuery) { + Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()), consoleQuery.release()); + } +} + +void TDynamicNameserver::ReplaceNameserverSetup(TIntrusivePtr newStaticConfig) { + if (StaticConfig->StaticNodeTable != newStaticConfig->StaticNodeTable) { + StaticConfig = std::move(newStaticConfig); + ListNodesCache->Invalidate(); + for (const auto& subscriber : StaticNodeChangeSubscribers) { + TActivationContext::Send(new IEventHandle(SelfId(), subscriber, new TEvInterconnect::TEvListNodes)); + } } } @@ -467,19 +480,13 @@ void TDynamicNameserver::Handle(TEvPrivate::TEvUpdateEpoch::TPtr &ev, const TAct void TDynamicNameserver::Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr /*ev*/) {} +void TDynamicNameserver::Handle(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse::TPtr /*ev*/) +{} + void TDynamicNameserver::Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr ev) { auto& record = ev->Get()->Record; - if (record.HasConfig()) { - if (const auto& config = record.GetConfig(); config.HasNameserviceConfig()) { - auto newStaticConfig = BuildNameserverTable(config.GetNameserviceConfig()); - if (StaticConfig->StaticNodeTable != newStaticConfig->StaticNodeTable) { - StaticConfig = std::move(newStaticConfig); - ListNodesCache->Invalidate(); - for (const auto& subscriber : StaticNodeChangeSubscribers) { - TActivationContext::Send(new IEventHandle(SelfId(), subscriber, new TEvInterconnect::TEvListNodes)); - } - } - } + if (SubscribedToConsole && record.HasConfig() && record.GetConfig().HasNameserviceConfig()) { + ReplaceNameserverSetup(BuildNameserverTable(record.GetConfig().GetNameserviceConfig())); } Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(record), 0, ev->Cookie); } diff --git a/ydb/core/mind/dynamic_nameserver_impl.h b/ydb/core/mind/dynamic_nameserver_impl.h index b86046c33da5..bc6dce956c95 100644 --- a/ydb/core/mind/dynamic_nameserver_impl.h +++ b/ydb/core/mind/dynamic_nameserver_impl.h @@ -232,6 +232,7 @@ class TDynamicNameserver : public TActorBootstrapped { hFunc(TEvents::TEvUnsubscribe, Handle); hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, Handle); + hFunc(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse, Handle); hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, Handle); hFunc(TEvNodeWardenStorageConfig, Handle); @@ -269,10 +270,13 @@ class TDynamicNameserver : public TActorBootstrapped { void Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorContext &ctx); void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr ev); + void Handle(NConsole::TEvConfigsDispatcher::TEvRemoveConfigSubscriptionResponse::TPtr ev); void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr ev); void Handle(TEvents::TEvUnsubscribe::TPtr ev); + void ReplaceNameserverSetup(TIntrusivePtr newStaticConfig); + private: TIntrusivePtr StaticConfig; std::array DynamicConfigs;