issue-1444: move netlink helpers to core/libs/netlink (#2471)

ydb-platform · Nov 15, 2024 · 50f14dd · 50f14dd
1 parent 98d2c20
commit 50f14dd
Show file tree

Hide file tree

Showing 46 changed files with 743 additions and 303 deletions.
diff --git a/cloud/blockstore/config/diagnostics.proto b/cloud/blockstore/config/diagnostics.proto
@@ -75,6 +75,15 @@ message TMonitoringUrlData
     optional string MonitoringNBSTVDashboard = 7;
 };
 
+////////////////////////////////////////////////////////////////////////////////
+// StatsFetcher type
+
+enum EStatsFetcherType
+{
+    CGROUP = 0;
+    KERNEL_TASK_DELAYACCT = 1;
+};
+
 ////////////////////////////////////////////////////////////////////////////////
 
 message TDiagnosticsConfig
@@ -216,4 +225,7 @@ message TDiagnosticsConfig
 
     // Performance measurements coefficients for local HDD disks.
     optional TVolumePerfSettings LocalHDDPerfSettings = 51;
+
+    // Type of fetching CPU stats
+    optional EStatsFetcherType StatsFetcherType = 52;
 }
diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.cpp b/cloud/blockstore/libs/daemon/common/bootstrap.cpp
@@ -861,7 +861,7 @@ void TBootstrapBase::Start()
     START_KIKIMR_COMPONENT(NotifyService);
     START_COMMON_COMPONENT(Monitoring);
     START_COMMON_COMPONENT(ProfileLog);
-    START_KIKIMR_COMPONENT(CgroupStatsFetcher);
+    START_KIKIMR_COMPONENT(StatsFetcher);
     START_COMMON_COMPONENT(DiscoveryService);
     START_COMMON_COMPONENT(TraceProcessor);
     START_KIKIMR_COMPONENT(TraceSerializer);
@@ -957,7 +957,7 @@ void TBootstrapBase::Stop()
     STOP_KIKIMR_COMPONENT(TraceSerializer);
     STOP_COMMON_COMPONENT(TraceProcessor);
     STOP_COMMON_COMPONENT(DiscoveryService);
-    STOP_KIKIMR_COMPONENT(CgroupStatsFetcher);
+    STOP_KIKIMR_COMPONENT(StatsFetcher);
     STOP_COMMON_COMPONENT(ProfileLog);
     STOP_COMMON_COMPONENT(Monitoring);
     STOP_KIKIMR_COMPONENT(LogbrokerService);

diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.h b/cloud/blockstore/libs/daemon/common/bootstrap.h
@@ -108,7 +108,7 @@ class TBootstrapBase
     virtual IStartable* GetTraceSerializer() = 0;
     virtual IStartable* GetLogbrokerService() = 0;
     virtual IStartable* GetNotifyService() = 0;
-    virtual IStartable* GetCgroupStatsFetcher() = 0;
+    virtual IStartable* GetStatsFetcher() = 0;
     virtual IStartable* GetIamTokenClient() = 0;
     virtual IStartable* GetComputeClient() = 0;
     virtual IStartable* GetKmsClient() = 0;

diff --git a/cloud/blockstore/libs/daemon/local/bootstrap.h b/cloud/blockstore/libs/daemon/local/bootstrap.h
@@ -32,7 +32,7 @@ class TBootstrapLocal final
     IStartable* GetTraceSerializer() override    { return nullptr; }
     IStartable* GetLogbrokerService() override   { return nullptr; }
     IStartable* GetNotifyService() override      { return nullptr; }
-    IStartable* GetCgroupStatsFetcher() override { return nullptr; }
+    IStartable* GetStatsFetcher() override       { return nullptr; }
     IStartable* GetIamTokenClient() override     { return nullptr; }
     IStartable* GetComputeClient() override      { return nullptr; }
     IStartable* GetKmsClient() override          { return nullptr; }

diff --git a/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp b/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp
@@ -89,6 +89,48 @@ NRdma::TClientConfigPtr CreateRdmaClientConfig(
     return std::make_shared<NRdma::TClientConfig>(config->GetClient());
 }
 
+// One can use either a service name and have the stats file inferred from it,
+ // or provide the stats file explicitly.
+ NCloud::NStorage::IStatsFetcherPtr BuildStatsFetcher(
+     NProto::EStatsFetcherType statsFetcherType,
+     const TString& cpuWaitServiceName,
+     const TString& cpuWaitFilename,
+     const TLog& log,
+     ILoggingServicePtr logging,
+     IMonitoringServicePtr monitoring,
+     TCgroupStatsFetcherMonitoringSettings cgroupStatsFetcherMonitoringSettings)
+ {
+     switch (statsFetcherType) {
+         case NProto::EStatsFetcherType::CGROUP: {
+             if (cpuWaitServiceName.Empty() && cpuWaitFilename.Empty()) {
+                 const auto& Log = log;
+                 STORAGE_INFO(
+                     "CpuWaitServiceName and CpuWaitFilename are empty, can't "
+                     "build "
+                     "CgroupStatsFetcher");
+                 return CreateStatsFetcherStub();
+             }
+             TString statsFile =
+                 cpuWaitFilename.Empty()
+                     ? NCloud::NStorage::BuildCpuWaitStatsFilename(
+                           cpuWaitServiceName)
+                     : cpuWaitFilename;
+
+             return CreateCgroupStatsFetcher(
+                 "FILESTORE_CGROUPS",
+                 std::move(logging),
+                 std::move(monitoring),
+                 statsFile,
+                 std::move(cgroupStatsFetcherMonitoringSettings));
+         }
+         case NProto::EStatsFetcherType::KERNEL_TASK_DELAYACCT:
+             return CreateKernelTaskDelayAcctStatsFetcher(
+                 "FILESTORE_KERNEL_TASK_DELAYACCT",
+                 std::move(logging),
+                 std::move(monitoring));
+     }
+ };
+
 }   // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -129,7 +171,7 @@ IStartable* TBootstrapYdb::GetYdbStorage()         { return YdbStorage.get(); }
 IStartable* TBootstrapYdb::GetTraceSerializer()    { return TraceSerializer.get(); }
 IStartable* TBootstrapYdb::GetLogbrokerService()   { return LogbrokerService.get(); }
 IStartable* TBootstrapYdb::GetNotifyService()      { return NotifyService.get(); }
-IStartable* TBootstrapYdb::GetCgroupStatsFetcher() { return CgroupStatsFetcher.get(); }
+IStartable* TBootstrapYdb::GetStatsFetcher()       { return StatsFetcher.get(); }
 IStartable* TBootstrapYdb::GetIamTokenClient()     { return IamTokenClient.get(); }
 IStartable* TBootstrapYdb::GetComputeClient()      { return ComputeClient.get(); }
 IStartable* TBootstrapYdb::GetKmsClient()          { return KmsClient.get(); }
@@ -492,11 +534,13 @@ void TBootstrapYdb::InitKikimrService()
             .CounterName = "CpuWaitFailure",
         };
 
-    CgroupStatsFetcher = CreateCgroupStatsFetcher(
-        "BLOCKSTORE_CGROUPS",
+    StatsFetcher = BuildStatsFetcher(
+        Configs->DiagnosticsConfig->GetStatsFetcherType(),
+        {},
+        Configs->DiagnosticsConfig->GetCpuWaitFilename(),
+        Log,
         logging,
         monitoring,
-        Configs->DiagnosticsConfig->GetCpuWaitFilename(),
         std::move(cgroupStatsFetcherMonitoringSettings));
 
     if (Configs->StorageConfig->GetBlockDigestsEnabled()) {
@@ -547,7 +591,7 @@ void TBootstrapYdb::InitKikimrService()
     args.LogbrokerService = LogbrokerService;
     args.NotifyService = NotifyService;
     args.VolumeStats = VolumeStats;
-    args.CgroupStatsFetcher = CgroupStatsFetcher;
+    args.StatsFetcher = StatsFetcher;
     args.RdmaServer = nullptr;
     args.RdmaClient = RdmaClient;
     args.Logging = logging;

diff --git a/cloud/blockstore/libs/daemon/ydb/bootstrap.h b/cloud/blockstore/libs/daemon/ydb/bootstrap.h
@@ -79,7 +79,7 @@ struct TBootstrapYdb final
     ITraceSerializerPtr TraceSerializer;
     NLogbroker::IServicePtr LogbrokerService;
     NNotify::IServicePtr NotifyService;
-    NCloud::NStorage::ICgroupStatsFetcherPtr CgroupStatsFetcher;
+    NCloud::NStorage::IStatsFetcherPtr StatsFetcher;
     NIamClient::IIamTokenClientPtr IamTokenClient;
     IComputeClientPtr ComputeClient;
     IKmsClientPtr KmsClient;
@@ -106,7 +106,7 @@ struct TBootstrapYdb final
     IStartable* GetTraceSerializer() override;
     IStartable* GetLogbrokerService() override;
     IStartable* GetNotifyService() override;
-    IStartable* GetCgroupStatsFetcher() override;
+    IStartable* GetStatsFetcher() override;
     IStartable* GetIamTokenClient() override;
     IStartable* GetComputeClient() override;
     IStartable* GetKmsClient() override;

diff --git a/cloud/blockstore/libs/diagnostics/config.cpp b/cloud/blockstore/libs/diagnostics/config.cpp
@@ -55,6 +55,7 @@ namespace {
     xxx(LocalHDDDowntimeThreshold,           TDuration,       TDuration::Seconds(15)                    )\
     xxx(ReportHistogramAsMultipleCounters,   bool,            true                                      )\
     xxx(ReportHistogramAsSingleCounter,      bool,            false                                     )\
+    xxx(StatsFetcherType,     NProto::EStatsFetcherType,      NProto::EStatsFetcherType::CGROUP         )\
 // BLOCKSTORE_DIAGNOSTICS_CONFIG
 
 #define BLOCKSTORE_DIAGNOSTICS_DECLARE_CONFIG(name, type, value)               \
@@ -307,3 +308,12 @@ void Out<NCloud::TRequestThresholds>(
 {
     OutRequestThresholds(out, value);
 }
+
+template <>
+void Out<NCloud::NBlockStore::NProto::EStatsFetcherType>(
+    IOutputStream& out,
+    NCloud::NBlockStore::NProto::EStatsFetcherType statsFetcherType)
+{
+    out << NCloud::NBlockStore::NProto::EStatsFetcherType_Name(
+        statsFetcherType);
+}
diff --git a/cloud/blockstore/libs/diagnostics/config.h b/cloud/blockstore/libs/diagnostics/config.h
@@ -155,6 +155,8 @@ class TDiagnosticsConfig
     TRequestThresholds GetRequestThresholds() const;
     EHistogramCounterOptions GetHistogramCounterOptions() const;
 
+    NProto::EStatsFetcherType GetStatsFetcherType() const;
+
     void Dump(IOutputStream& out) const;
     void DumpHtml(IOutputStream& out) const;
 };