diff --git a/cloud/blockstore/config/diagnostics.proto b/cloud/blockstore/config/diagnostics.proto index 18f1c9a39ef..0eb4af531bd 100644 --- a/cloud/blockstore/config/diagnostics.proto +++ b/cloud/blockstore/config/diagnostics.proto @@ -75,6 +75,15 @@ message TMonitoringUrlData optional string MonitoringNBSTVDashboard = 7; }; +//////////////////////////////////////////////////////////////////////////////// +// StatsFetcher type + +enum EStatsFetcherType +{ + CGROUP = 0; + KERNEL_TASK_DELAYACCT = 1; +}; + //////////////////////////////////////////////////////////////////////////////// message TDiagnosticsConfig @@ -216,4 +225,7 @@ message TDiagnosticsConfig // Performance measurements coefficients for local HDD disks. optional TVolumePerfSettings LocalHDDPerfSettings = 51; + + // Type of fetching CPU stats + optional EStatsFetcherType StatsFetcherType = 52; } diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.cpp b/cloud/blockstore/libs/daemon/common/bootstrap.cpp index e93ecd3d6b8..75140e40fef 100644 --- a/cloud/blockstore/libs/daemon/common/bootstrap.cpp +++ b/cloud/blockstore/libs/daemon/common/bootstrap.cpp @@ -861,7 +861,7 @@ void TBootstrapBase::Start() START_KIKIMR_COMPONENT(NotifyService); START_COMMON_COMPONENT(Monitoring); START_COMMON_COMPONENT(ProfileLog); - START_KIKIMR_COMPONENT(CgroupStatsFetcher); + START_KIKIMR_COMPONENT(StatsFetcher); START_COMMON_COMPONENT(DiscoveryService); START_COMMON_COMPONENT(TraceProcessor); START_KIKIMR_COMPONENT(TraceSerializer); @@ -957,7 +957,7 @@ void TBootstrapBase::Stop() STOP_KIKIMR_COMPONENT(TraceSerializer); STOP_COMMON_COMPONENT(TraceProcessor); STOP_COMMON_COMPONENT(DiscoveryService); - STOP_KIKIMR_COMPONENT(CgroupStatsFetcher); + STOP_KIKIMR_COMPONENT(StatsFetcher); STOP_COMMON_COMPONENT(ProfileLog); STOP_COMMON_COMPONENT(Monitoring); STOP_KIKIMR_COMPONENT(LogbrokerService); diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.h b/cloud/blockstore/libs/daemon/common/bootstrap.h index 27fbec848c1..8fa7800337b 100644 --- a/cloud/blockstore/libs/daemon/common/bootstrap.h +++ b/cloud/blockstore/libs/daemon/common/bootstrap.h @@ -108,7 +108,7 @@ class TBootstrapBase virtual IStartable* GetTraceSerializer() = 0; virtual IStartable* GetLogbrokerService() = 0; virtual IStartable* GetNotifyService() = 0; - virtual IStartable* GetCgroupStatsFetcher() = 0; + virtual IStartable* GetStatsFetcher() = 0; virtual IStartable* GetIamTokenClient() = 0; virtual IStartable* GetComputeClient() = 0; virtual IStartable* GetKmsClient() = 0; diff --git a/cloud/blockstore/libs/daemon/local/bootstrap.h b/cloud/blockstore/libs/daemon/local/bootstrap.h index 86f70729010..bf963ee3bcc 100644 --- a/cloud/blockstore/libs/daemon/local/bootstrap.h +++ b/cloud/blockstore/libs/daemon/local/bootstrap.h @@ -32,7 +32,7 @@ class TBootstrapLocal final IStartable* GetTraceSerializer() override { return nullptr; } IStartable* GetLogbrokerService() override { return nullptr; } IStartable* GetNotifyService() override { return nullptr; } - IStartable* GetCgroupStatsFetcher() override { return nullptr; } + IStartable* GetStatsFetcher() override { return nullptr; } IStartable* GetIamTokenClient() override { return nullptr; } IStartable* GetComputeClient() override { return nullptr; } IStartable* GetKmsClient() override { return nullptr; } diff --git a/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp b/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp index 68ee6dda56e..e43f8bb9a1b 100644 --- a/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp +++ b/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp @@ -89,6 +89,48 @@ NRdma::TClientConfigPtr CreateRdmaClientConfig( return std::make_shared(config->GetClient()); } +// One can use either a service name and have the stats file inferred from it, + // or provide the stats file explicitly. + NCloud::NStorage::IStatsFetcherPtr BuildStatsFetcher( + NProto::EStatsFetcherType statsFetcherType, + const TString& cpuWaitServiceName, + const TString& cpuWaitFilename, + const TLog& log, + ILoggingServicePtr logging, + IMonitoringServicePtr monitoring, + TCgroupStatsFetcherMonitoringSettings cgroupStatsFetcherMonitoringSettings) + { + switch (statsFetcherType) { + case NProto::EStatsFetcherType::CGROUP: { + if (cpuWaitServiceName.Empty() && cpuWaitFilename.Empty()) { + const auto& Log = log; + STORAGE_INFO( + "CpuWaitServiceName and CpuWaitFilename are empty, can't " + "build " + "CgroupStatsFetcher"); + return CreateStatsFetcherStub(); + } + TString statsFile = + cpuWaitFilename.Empty() + ? NCloud::NStorage::BuildCpuWaitStatsFilename( + cpuWaitServiceName) + : cpuWaitFilename; + + return CreateCgroupStatsFetcher( + "FILESTORE_CGROUPS", + std::move(logging), + std::move(monitoring), + statsFile, + std::move(cgroupStatsFetcherMonitoringSettings)); + } + case NProto::EStatsFetcherType::KERNEL_TASK_DELAYACCT: + return CreateKernelTaskDelayAcctStatsFetcher( + "FILESTORE_KERNEL_TASK_DELAYACCT", + std::move(logging), + std::move(monitoring)); + } + }; + } // namespace //////////////////////////////////////////////////////////////////////////////// @@ -129,7 +171,7 @@ IStartable* TBootstrapYdb::GetYdbStorage() { return YdbStorage.get(); } IStartable* TBootstrapYdb::GetTraceSerializer() { return TraceSerializer.get(); } IStartable* TBootstrapYdb::GetLogbrokerService() { return LogbrokerService.get(); } IStartable* TBootstrapYdb::GetNotifyService() { return NotifyService.get(); } -IStartable* TBootstrapYdb::GetCgroupStatsFetcher() { return CgroupStatsFetcher.get(); } +IStartable* TBootstrapYdb::GetStatsFetcher() { return StatsFetcher.get(); } IStartable* TBootstrapYdb::GetIamTokenClient() { return IamTokenClient.get(); } IStartable* TBootstrapYdb::GetComputeClient() { return ComputeClient.get(); } IStartable* TBootstrapYdb::GetKmsClient() { return KmsClient.get(); } @@ -492,11 +534,13 @@ void TBootstrapYdb::InitKikimrService() .CounterName = "CpuWaitFailure", }; - CgroupStatsFetcher = CreateCgroupStatsFetcher( - "BLOCKSTORE_CGROUPS", + StatsFetcher = BuildStatsFetcher( + Configs->DiagnosticsConfig->GetStatsFetcherType(), + {}, + Configs->DiagnosticsConfig->GetCpuWaitFilename(), + Log, logging, monitoring, - Configs->DiagnosticsConfig->GetCpuWaitFilename(), std::move(cgroupStatsFetcherMonitoringSettings)); if (Configs->StorageConfig->GetBlockDigestsEnabled()) { @@ -547,7 +591,7 @@ void TBootstrapYdb::InitKikimrService() args.LogbrokerService = LogbrokerService; args.NotifyService = NotifyService; args.VolumeStats = VolumeStats; - args.CgroupStatsFetcher = CgroupStatsFetcher; + args.StatsFetcher = StatsFetcher; args.RdmaServer = nullptr; args.RdmaClient = RdmaClient; args.Logging = logging; diff --git a/cloud/blockstore/libs/daemon/ydb/bootstrap.h b/cloud/blockstore/libs/daemon/ydb/bootstrap.h index 9737f16b28e..9ca8262eefc 100644 --- a/cloud/blockstore/libs/daemon/ydb/bootstrap.h +++ b/cloud/blockstore/libs/daemon/ydb/bootstrap.h @@ -79,7 +79,7 @@ struct TBootstrapYdb final ITraceSerializerPtr TraceSerializer; NLogbroker::IServicePtr LogbrokerService; NNotify::IServicePtr NotifyService; - NCloud::NStorage::ICgroupStatsFetcherPtr CgroupStatsFetcher; + NCloud::NStorage::IStatsFetcherPtr StatsFetcher; NIamClient::IIamTokenClientPtr IamTokenClient; IComputeClientPtr ComputeClient; IKmsClientPtr KmsClient; @@ -106,7 +106,7 @@ struct TBootstrapYdb final IStartable* GetTraceSerializer() override; IStartable* GetLogbrokerService() override; IStartable* GetNotifyService() override; - IStartable* GetCgroupStatsFetcher() override; + IStartable* GetStatsFetcher() override; IStartable* GetIamTokenClient() override; IStartable* GetComputeClient() override; IStartable* GetKmsClient() override; diff --git a/cloud/blockstore/libs/diagnostics/config.cpp b/cloud/blockstore/libs/diagnostics/config.cpp index d7e850d9523..8fef2dd8894 100644 --- a/cloud/blockstore/libs/diagnostics/config.cpp +++ b/cloud/blockstore/libs/diagnostics/config.cpp @@ -55,6 +55,7 @@ namespace { xxx(LocalHDDDowntimeThreshold, TDuration, TDuration::Seconds(15) )\ xxx(ReportHistogramAsMultipleCounters, bool, true )\ xxx(ReportHistogramAsSingleCounter, bool, false )\ + xxx(StatsFetcherType, NProto::EStatsFetcherType, NProto::EStatsFetcherType::CGROUP )\ // BLOCKSTORE_DIAGNOSTICS_CONFIG #define BLOCKSTORE_DIAGNOSTICS_DECLARE_CONFIG(name, type, value) \ @@ -307,3 +308,12 @@ void Out( { OutRequestThresholds(out, value); } + +template <> +void Out( + IOutputStream& out, + NCloud::NBlockStore::NProto::EStatsFetcherType statsFetcherType) +{ + out << NCloud::NBlockStore::NProto::EStatsFetcherType_Name( + statsFetcherType); +} diff --git a/cloud/blockstore/libs/diagnostics/config.h b/cloud/blockstore/libs/diagnostics/config.h index e4133241692..e00fd0bfc62 100644 --- a/cloud/blockstore/libs/diagnostics/config.h +++ b/cloud/blockstore/libs/diagnostics/config.h @@ -155,6 +155,8 @@ class TDiagnosticsConfig TRequestThresholds GetRequestThresholds() const; EHistogramCounterOptions GetHistogramCounterOptions() const; + NProto::EStatsFetcherType GetStatsFetcherType() const; + void Dump(IOutputStream& out) const; void DumpHtml(IOutputStream& out) const; }; diff --git a/cloud/blockstore/libs/nbd/netlink_device.cpp b/cloud/blockstore/libs/nbd/netlink_device.cpp index 6e9bd52779f..930de1c6d7c 100644 --- a/cloud/blockstore/libs/nbd/netlink_device.cpp +++ b/cloud/blockstore/libs/nbd/netlink_device.cpp @@ -3,13 +3,10 @@ #include "utils.h" #include +#include #include -#include -#include -#include - #include #include @@ -19,177 +16,18 @@ namespace { using namespace NThreading; -using TResponseHandler = std::function; - //////////////////////////////////////////////////////////////////////////////// constexpr TStringBuf NBD_DEVICE_SUFFIX = "/dev/nbd"; //////////////////////////////////////////////////////////////////////////////// -class TNetlinkSocket -{ -private: - nl_sock* Socket; - int Family; - -public: - TNetlinkSocket() - { - Socket = nl_socket_alloc(); - - if (Socket == nullptr) { - throw TServiceError(E_FAIL) << "unable to allocate netlink socket"; - } - - if (int err = genl_connect(Socket)) { - nl_socket_free(Socket); - throw TServiceError(E_FAIL) - << "unable to connect to generic netlink socket: " - << nl_geterror(err); - } - - Family = genl_ctrl_resolve(Socket, "nbd"); - - if (Family < 0) { - nl_socket_free(Socket); - throw TServiceError(E_FAIL) - << "unable to resolve nbd netlink family: " - << nl_geterror(Family); - } - } - - ~TNetlinkSocket() - { - nl_socket_free(Socket); - } - - int GetFamily() const - { - return Family; - } - - template - void SetCallback(nl_cb_type type, F func) - { - auto arg = std::make_unique(std::move(func)); - - if (int err = nl_socket_modify_cb( - Socket, - type, - NL_CB_CUSTOM, - TNetlinkSocket::ResponseHandler, - arg.get())) - { - throw TServiceError(E_FAIL) - << "unable to set socket callback: " << nl_geterror(err); - } - arg.release(); - } - - static int ResponseHandler(nl_msg* msg, void* arg) - { - auto func = std::unique_ptr( - static_cast(arg)); - - return (*func)(static_cast(nlmsg_data(nlmsg_hdr(msg)))); - } - - void Send(nl_msg* message) - { - if (int err = nl_send_auto(Socket, message); err < 0) { - throw TServiceError(E_FAIL) - << "send error: " << nl_geterror(err); - } - if (int err = nl_wait_for_ack(Socket)) { - // this is either recv error, or an actual error message received - // from the kernel - throw TServiceError(E_FAIL) - << "recv error: " << nl_geterror(err); - } - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TNestedAttribute -{ -private: - nl_msg* Message; - nlattr* Attribute; - -public: - TNestedAttribute(nl_msg* message, int attribute) - : Message(message) - { - Attribute = nla_nest_start(message, attribute); - if (!Attribute) { - throw TServiceError(E_FAIL) << "unable to nest attribute"; - } - } - - ~TNestedAttribute() - { - nla_nest_end(Message, Attribute); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TNetlinkMessage -{ -private: - nl_msg* Message; - -public: - TNetlinkMessage(int family, int command) - { - Message = nlmsg_alloc(); - if (Message == nullptr) { - throw TServiceError(E_FAIL) << "unable to allocate message"; - } - genlmsg_put( - Message, - NL_AUTO_PORT, - NL_AUTO_SEQ, - family, - 0, // hdrlen - 0, // flags - command, - 0); // version - } - - ~TNetlinkMessage() - { - nlmsg_free(Message); - } - - operator nl_msg*() const - { - return Message; - } - - template - void Put(int attribute, T data) - { - if (int err = nla_put(Message, attribute, sizeof(T), &data)) { - throw TServiceError(E_FAIL) << "unable to put attribute " - << attribute << ": " << nl_geterror(err); - } - } - - TNestedAttribute Nest(int attribute) - { - return TNestedAttribute(Message, attribute); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - class TNetlinkDevice final : public IDevice , public std::enable_shared_from_this { + using TNetlinkMessage = NCloud::NNetlink::TMessage; + using TNetlinkSocket = NCloud::NNetlink::TSocket; private: const ILoggingServicePtr Logging; const TNetworkAddress ConnectAddress; @@ -231,7 +69,7 @@ class TNetlinkDevice final void Disconnect(); void DoConnect(bool connected); - int StatusHandler(genlmsghdr* header); + int StatusHandler(nl_msg* nlmsg); }; //////////////////////////////////////////////////////////////////////////////// @@ -309,7 +147,7 @@ TFuture TNetlinkDevice::Stop(bool deleteDevice) TFuture TNetlinkDevice::Resize(ui64 deviceSizeInBytes) { try { - TNetlinkSocket socket; + TNetlinkSocket socket("nbd"); TNetlinkMessage message(socket.GetFamily(), NBD_CMD_RECONFIGURE); message.Put(NBD_ATTR_INDEX, DeviceIndex); @@ -373,14 +211,14 @@ void TNetlinkDevice::DisconnectSocket() // or reconfigure (if Reconfigure == true) specified device void TNetlinkDevice::Connect() { - TNetlinkSocket socket; + TNetlinkSocket socket("nbd"); socket.SetCallback( NL_CB_VALID, - [device = shared_from_this()] (auto* header) { - return device->StatusHandler(header); + [device = shared_from_this()](auto* nlmsg) { + return device->StatusHandler(nlmsg); }); - TNetlinkMessage message(socket.GetFamily(), NBD_CMD_STATUS); + NCloud::NNetlink::TNetlinkMessage message(socket.GetFamily(), NBD_CMD_STATUS); message.Put(NBD_ATTR_INDEX, DeviceIndex); socket.Send(message); } @@ -389,7 +227,7 @@ void TNetlinkDevice::Disconnect() { STORAGE_INFO("disconnect " << DeviceName); - TNetlinkSocket socket; + TNetlinkSocket socket("nbd"); TNetlinkMessage message(socket.GetFamily(), NBD_CMD_DISCONNECT); message.Put(NBD_ATTR_INDEX, DeviceIndex); socket.Send(message); @@ -410,7 +248,7 @@ void TNetlinkDevice::DoConnect(bool connected) STORAGE_INFO("connect " << DeviceName); } - TNetlinkSocket socket; + TNetlinkSocket socket("nbd"); TNetlinkMessage message(socket.GetFamily(), command); const auto& info = Handler->GetExportInfo(); @@ -448,8 +286,9 @@ void TNetlinkDevice::DoConnect(bool connected) } } -int TNetlinkDevice::StatusHandler(genlmsghdr* header) +int TNetlinkDevice::StatusHandler(nl_msg* nlmsg) { + auto header = static_cast(nlmsg_data(nlmsg_hdr(nlmsg))); nlattr* attr[NBD_ATTR_MAX + 1] = {}; nlattr* deviceItem[NBD_DEVICE_ITEM_MAX + 1] = {}; nlattr* device[NBD_DEVICE_ATTR_MAX + 1] = {}; diff --git a/cloud/blockstore/libs/nbd/ya.make b/cloud/blockstore/libs/nbd/ya.make index 01d3adf187a..d6340b22e73 100644 --- a/cloud/blockstore/libs/nbd/ya.make +++ b/cloud/blockstore/libs/nbd/ya.make @@ -1,10 +1,5 @@ LIBRARY() -LICENSE_RESTRICTION_EXCEPTIONS( - contrib/restricted/libnl/lib/nl-3 - contrib/restricted/libnl/lib/nl-genl-3 -) - SRCS( binary_reader.cpp binary_writer.cpp @@ -30,9 +25,8 @@ PEERDIR( cloud/blockstore/libs/diagnostics cloud/blockstore/libs/service cloud/storage/core/libs/coroutine + cloud/storage/core/libs/netlink contrib/libs/linux-headers - contrib/restricted/libnl/lib/nl-3 - contrib/restricted/libnl/lib/nl-genl-3 library/cpp/coroutine/engine library/cpp/coroutine/listener library/cpp/deprecated/atomic diff --git a/cloud/blockstore/libs/storage/init/server/actorsystem.cpp b/cloud/blockstore/libs/storage/init/server/actorsystem.cpp index 16b1f9cb488..0e38981f140 100644 --- a/cloud/blockstore/libs/storage/init/server/actorsystem.cpp +++ b/cloud/blockstore/libs/storage/init/server/actorsystem.cpp @@ -307,7 +307,7 @@ class TStorageServicesInitializer final auto volumeBalancerService = CreateVolumeBalancerActor( Args.StorageConfig, Args.VolumeStats, - Args.CgroupStatsFetcher, + Args.StatsFetcher, Args.VolumeBalancerSwitch, MakeStorageServiceId()); diff --git a/cloud/blockstore/libs/storage/init/server/actorsystem.h b/cloud/blockstore/libs/storage/init/server/actorsystem.h index 2b215f3c2be..bbf967e2c50 100644 --- a/cloud/blockstore/libs/storage/init/server/actorsystem.h +++ b/cloud/blockstore/libs/storage/init/server/actorsystem.h @@ -60,7 +60,7 @@ struct TServerActorSystemArgs IVolumeStatsPtr VolumeStats; NRdma::IServerPtr RdmaServer; NRdma::IClientPtr RdmaClient; - NCloud::NStorage::ICgroupStatsFetcherPtr CgroupStatsFetcher; + NCloud::NStorage::IStatsFetcherPtr StatsFetcher; TManuallyPreemptedVolumesPtr PreemptedVolumes; NNvme::INvmeManagerPtr NvmeManager; IVolumeBalancerSwitchPtr VolumeBalancerSwitch; diff --git a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer.cpp b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer.cpp index 68f688cdae6..d4a7cf2d845 100644 --- a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer.cpp +++ b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer.cpp @@ -11,14 +11,14 @@ using namespace NActors; IActorPtr CreateVolumeBalancerActor( TStorageConfigPtr storageConfig, IVolumeStatsPtr volumeStats, - NCloud::NStorage::ICgroupStatsFetcherPtr cgroupStatFetcher, + NCloud::NStorage::IStatsFetcherPtr statFetcher, IVolumeBalancerSwitchPtr volumeBalancerSwitch, NActors::TActorId serviceActorId) { return std::make_unique( std::move(storageConfig), std::move(volumeStats), - std::move(cgroupStatFetcher), + std::move(statFetcher), std::move(volumeBalancerSwitch), serviceActorId); } diff --git a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer.h b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer.h index 539448faf4a..289b4e76734 100644 --- a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer.h +++ b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer.h @@ -20,7 +20,7 @@ namespace NCloud::NBlockStore::NStorage { NActors::IActorPtr CreateVolumeBalancerActor( TStorageConfigPtr storageConfig, IVolumeStatsPtr volumeStats, - NCloud::NStorage::ICgroupStatsFetcherPtr cgroupStatFetcher, + NCloud::NStorage::IStatsFetcherPtr cgroupStatFetcher, IVolumeBalancerSwitchPtr volumeBalancerSwitch, NActors::TActorId serviceActorId); diff --git a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_actor.cpp b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_actor.cpp index 95d380f66ae..3105c7c7807 100644 --- a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_actor.cpp +++ b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_actor.cpp @@ -140,12 +140,12 @@ STFUNC(TRemoteVolumeStatActor::StateWork) TVolumeBalancerActor::TVolumeBalancerActor( TStorageConfigPtr storageConfig, IVolumeStatsPtr volumeStats, - NCloud::NStorage::ICgroupStatsFetcherPtr cgroupStatsFetcher, + NCloud::NStorage::IStatsFetcherPtr statsFetcher, IVolumeBalancerSwitchPtr volumeBalancerSwitch, TActorId serviceActorId) : StorageConfig(std::move(storageConfig)) , VolumeStats(std::move(volumeStats)) - , CgroupStatsFetcher(std::move(cgroupStatsFetcher)) + , StatsFetcher(std::move(statsFetcher)) , VolumeBalancerSwitch(std::move(volumeBalancerSwitch)) , ServiceActorId(serviceActorId) , State(std::make_unique(StorageConfig)) @@ -245,7 +245,14 @@ void TVolumeBalancerActor::HandleGetVolumeStatsResponse( auto now = ctx.Now(); auto interval = (now - LastCpuWaitQuery).MicroSeconds(); - auto cpuLack = CpuLackPercentsMultiplier * CgroupStatsFetcher->GetCpuWait().MicroSeconds(); + auto cpuWait = StatsFetcher->GetCpuWait(); + if (HasError(cpuWait)) { + LOG_ERROR_S(ctx, TBlockStoreComponents::VOLUME_BALANCER, + "Failed to get cpu wait: " << cpuWait.GetError()); + } + auto cpuLack = + CpuLackPercentsMultiplier * + (HasError(cpuWait) ? 0 : cpuWait.GetResult().MicroSeconds()); cpuLack /= interval; *CpuWait = cpuLack; diff --git a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_actor.h b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_actor.h index b0589e35bc3..fb42500d8e6 100644 --- a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_actor.h +++ b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_actor.h @@ -28,7 +28,7 @@ class TVolumeBalancerActor final private: const TStorageConfigPtr StorageConfig; const IVolumeStatsPtr VolumeStats; - const NCloud::NStorage::ICgroupStatsFetcherPtr CgroupStatsFetcher; + const NCloud::NStorage::IStatsFetcherPtr StatsFetcher; const IVolumeBalancerSwitchPtr VolumeBalancerSwitch; const NActors::TActorId ServiceActorId; @@ -49,7 +49,7 @@ class TVolumeBalancerActor final TVolumeBalancerActor( TStorageConfigPtr storageConfig, IVolumeStatsPtr volumeStats, - NCloud::NStorage::ICgroupStatsFetcherPtr cgroupStatsFetcher, + NCloud::NStorage::IStatsFetcherPtr statsFetcher, IVolumeBalancerSwitchPtr volumeBalancerSwitch, NActors::TActorId serviceActorId); diff --git a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_ut.cpp b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_ut.cpp index 3928bf6d6ee..7cb487280d5 100644 --- a/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_ut.cpp +++ b/cloud/blockstore/libs/storage/volume_balancer/volume_balancer_ut.cpp @@ -194,7 +194,7 @@ struct TVolumeStatsTestMock final //////////////////////////////////////////////////////////////////////////////// -struct TCgroupStatsFetcherMock: public NCloud::NStorage::ICgroupStatsFetcher +struct TStatsFetcherMock: public NCloud::NStorage::IStatsFetcher { TDuration Value; @@ -211,7 +211,7 @@ struct TCgroupStatsFetcherMock: public NCloud::NStorage::ICgroupStatsFetcher { } - TDuration GetCpuWait() override + TResultOrError GetCpuWait() override { return Value; }; @@ -230,14 +230,14 @@ class TVolumeBalancerTestEnv public: std::shared_ptr VolumeStats; - std::shared_ptr Fetcher; + std::shared_ptr Fetcher; public: TVolumeBalancerTestEnv() { Sender = TestEnv.GetRuntime().AllocateEdgeActor(); VolumeStats = std::make_shared(); - Fetcher = std::make_shared(); + Fetcher = std::make_shared(); } TActorId GetEdgeActor() const @@ -411,7 +411,7 @@ NFeatures::TFeaturesConfigPtr CreateFeatureConfig( IActorPtr CreateVolumeBalancerActor( TVolumeBalancerConfigBuilder& config, IVolumeStatsPtr volumeStats, - NCloud::NStorage::ICgroupStatsFetcherPtr cgroupStatsFetcher, + NCloud::NStorage::IStatsFetcherPtr statsFetcher, TActorId serviceActorId) { NProto::TStorageServiceConfig storageConfig = config.Build(); @@ -425,7 +425,7 @@ IActorPtr CreateVolumeBalancerActor( CreateFeatureConfig("Balancer", {}) ), std::move(volumeStats), - std::move(cgroupStatsFetcher), + std::move(statsFetcher), std::move(volumeBalancerSwitch), std::move(serviceActorId)); } diff --git a/cloud/filestore/config/diagnostics.proto b/cloud/filestore/config/diagnostics.proto index 38596d19aa5..cf175505ccc 100644 --- a/cloud/filestore/config/diagnostics.proto +++ b/cloud/filestore/config/diagnostics.proto @@ -19,6 +19,15 @@ message TMonitoringUrlData optional string MonitoringProject = 3; }; +//////////////////////////////////////////////////////////////////////////////// +// StatsFetcher type + +enum EStatsFetcherType +{ + CGROUP = 0; + KERNEL_TASK_DELAYACCT = 1; +}; + //////////////////////////////////////////////////////////////////////////////// message TDiagnosticsConfig @@ -103,4 +112,7 @@ message TDiagnosticsConfig // Report histogram as a single counter (THistogramCounter) optional bool ReportHistogramAsSingleCounter = 25; + + // Type of fetching CPU stats + optional EStatsFetcherType StatsFetcherType = 26; } diff --git a/cloud/filestore/libs/daemon/common/bootstrap.cpp b/cloud/filestore/libs/daemon/common/bootstrap.cpp index 8e8b8696199..f62b1095259 100644 --- a/cloud/filestore/libs/daemon/common/bootstrap.cpp +++ b/cloud/filestore/libs/daemon/common/bootstrap.cpp @@ -57,7 +57,8 @@ const TString SlowRequestsFilterId = "st_slow_requests_filter"; // One can use either a service name and have the stats file inferred from it, // or provide the stats file explicitly. -NCloud::NStorage::ICgroupStatsFetcherPtr BuildCgroupStatsFetcher( +NCloud::NStorage::IStatsFetcherPtr BuildStatsFetcher( + NProto::EStatsFetcherType statsFetcherType, const TString& cpuWaitServiceName, const TString& cpuWaitFilename, const TLog& log, @@ -65,30 +66,41 @@ NCloud::NStorage::ICgroupStatsFetcherPtr BuildCgroupStatsFetcher( IMonitoringServicePtr monitoring, const TString& metricsComponent) { - if (cpuWaitServiceName.empty() && cpuWaitFilename.empty()) { - const auto& Log = log; - STORAGE_INFO( - "CpuWaitServiceName and CpuWaitFilename are empty, can't build " - "CgroupStatsFetcher"); - return CreateCgroupStatsFetcherStub(); + switch (statsFetcherType) { + case NProto::EStatsFetcherType::CGROUP: { + if (cpuWaitServiceName.Empty() && cpuWaitFilename.Empty()) { + const auto& Log = log; + STORAGE_INFO( + "CpuWaitServiceName and CpuWaitFilename are empty, can't " + "build " + "CgroupStatsFetcher"); + return CreateStatsFetcherStub(); + } + auto cgroupStatsFetcherMonitoringSettings = + TCgroupStatsFetcherMonitoringSettings{ + .CountersGroupName = "filestore", + .ComponentGroupName = metricsComponent, + .CounterName = "CpuWaitFailure", + }; + TString statsFile = + cpuWaitFilename.Empty() + ? NCloud::NStorage::BuildCpuWaitStatsFilename( + cpuWaitServiceName) + : cpuWaitFilename; + + return CreateCgroupStatsFetcher( + "FILESTORE_CGROUPS", + std::move(logging), + std::move(monitoring), + statsFile, + std::move(cgroupStatsFetcherMonitoringSettings)); + } + case NProto::EStatsFetcherType::KERNEL_TASK_DELAYACCT: + return CreateKernelTaskDelayAcctStatsFetcher( + "FILESTORE_KERNEL_TASK_DELAYACCT", + std::move(logging), + std::move(monitoring)); } - auto cgroupStatsFetcherMonitoringSettings = - TCgroupStatsFetcherMonitoringSettings{ - .CountersGroupName = "filestore", - .ComponentGroupName = metricsComponent, - .CounterName = "CpuWaitFailure", - }; - TString statsFile = - cpuWaitFilename.empty() - ? NCloud::NStorage::BuildCpuWaitStatsFilename(cpuWaitServiceName) - : cpuWaitFilename; - - return CreateCgroupStatsFetcher( - "FILESTORE_CGROUPS", - std::move(logging), - std::move(monitoring), - statsFile, - std::move(cgroupStatsFetcherMonitoringSettings)); }; } // namespace @@ -126,7 +138,7 @@ void TBootstrapCommon::Start() FILESTORE_LOG_START_COMPONENT(BackgroundThreadPool); FILESTORE_LOG_START_COMPONENT(ProfileLog); FILESTORE_LOG_START_COMPONENT(RequestStatsUpdater); - FILESTORE_LOG_START_COMPONENT(CgroupStatsFetcher); + FILESTORE_LOG_START_COMPONENT(StatsFetcher); StartComponents(); @@ -155,7 +167,7 @@ void TBootstrapCommon::Stop() StopComponents(); - FILESTORE_LOG_STOP_COMPONENT(CgroupStatsFetcher); + FILESTORE_LOG_STOP_COMPONENT(StatsFetcher); FILESTORE_LOG_STOP_COMPONENT(RequestStatsUpdater); FILESTORE_LOG_STOP_COMPONENT(ProfileLog); FILESTORE_LOG_STOP_COMPONENT(BackgroundThreadPool); @@ -306,7 +318,8 @@ void TBootstrapCommon::InitActorSystem() STORAGE_INFO("TraceSerializer initialized"); - CgroupStatsFetcher = BuildCgroupStatsFetcher( + StatsFetcher = BuildStatsFetcher( + Configs->DiagnosticsConfig->GetStatsFetcherType(), Configs->DiagnosticsConfig->GetCpuWaitServiceName(), Configs->DiagnosticsConfig->GetCpuWaitFilename(), Log, @@ -314,7 +327,7 @@ void TBootstrapCommon::InitActorSystem() monitoring, MetricsComponent); - STORAGE_INFO("CgroupStatsFetcher initialized"); + STORAGE_INFO("StatsFetcher initialized"); NStorage::TActorSystemArgs args; args.NodeId = nodeId; @@ -326,7 +339,7 @@ void TBootstrapCommon::InitActorSystem() args.DiagnosticsConfig = Configs->DiagnosticsConfig; args.Metrics = Metrics; args.UserCounters = UserCounters; - args.CgroupStatsFetcher = CgroupStatsFetcher; + args.StatsFetcher = StatsFetcher; args.ModuleFactories = ModuleFactories; ActorSystem = NStorage::CreateActorSystem(args); diff --git a/cloud/filestore/libs/daemon/common/bootstrap.h b/cloud/filestore/libs/daemon/common/bootstrap.h index 634d2969b78..1172d53cd4e 100644 --- a/cloud/filestore/libs/daemon/common/bootstrap.h +++ b/cloud/filestore/libs/daemon/common/bootstrap.h @@ -73,7 +73,7 @@ class TBootstrapCommon ITaskQueuePtr BackgroundThreadPool; IProfileLogPtr ProfileLog; IActorSystemPtr ActorSystem; - NCloud::NStorage::ICgroupStatsFetcherPtr CgroupStatsFetcher; + NCloud::NStorage::IStatsFetcherPtr StatsFetcher; public: TBootstrapCommon( diff --git a/cloud/filestore/libs/diagnostics/config.cpp b/cloud/filestore/libs/diagnostics/config.cpp index e3ffc4670fd..9aa5a768bd5 100644 --- a/cloud/filestore/libs/diagnostics/config.cpp +++ b/cloud/filestore/libs/diagnostics/config.cpp @@ -37,6 +37,7 @@ namespace { xxx(MonitoringUrlData, TMonitoringUrlData, {} )\ xxx(ReportHistogramAsMultipleCounters, bool, true )\ xxx(ReportHistogramAsSingleCounter, bool, false )\ + xxx(StatsFetcherType, NProto::EStatsFetcherType, NProto::EStatsFetcherType::CGROUP )\ // FILESTORE_DIAGNOSTICS_CONFIG #define FILESTORE_DIAGNOSTICS_DECLARE_CONFIG(name, type, value) \ @@ -170,3 +171,12 @@ void Out( v.SetMonitoringProject(value.MonitoringProject); SerializeToTextFormat(v, out); } + +template <> +void Out( + IOutputStream& out, + NCloud::NFileStore::NProto::EStatsFetcherType statsFetcherType) +{ + out << NCloud::NFileStore::NProto::EStatsFetcherType_Name( + statsFetcherType); +} diff --git a/cloud/filestore/libs/diagnostics/config.h b/cloud/filestore/libs/diagnostics/config.h index a774690a3dc..2673875da4d 100644 --- a/cloud/filestore/libs/diagnostics/config.h +++ b/cloud/filestore/libs/diagnostics/config.h @@ -71,6 +71,8 @@ class TDiagnosticsConfig bool GetReportHistogramAsSingleCounter() const; EHistogramCounterOptions GetHistogramCounterOptions() const; + NProto::EStatsFetcherType GetStatsFetcherType() const; + void Dump(IOutputStream& out) const; void DumpHtml(IOutputStream& out) const; }; diff --git a/cloud/filestore/libs/storage/init/actorsystem.cpp b/cloud/filestore/libs/storage/init/actorsystem.cpp index c29cd93c766..5d2a137159d 100644 --- a/cloud/filestore/libs/storage/init/actorsystem.cpp +++ b/cloud/filestore/libs/storage/init/actorsystem.cpp @@ -85,7 +85,7 @@ class TStorageServicesInitializer final StatsRegistry, Args.ProfileLog, Args.TraceSerializer, - Args.CgroupStatsFetcher); + Args.StatsFetcher); setup->LocalServices.emplace_back( MakeStorageServiceId(), diff --git a/cloud/filestore/libs/storage/init/actorsystem.h b/cloud/filestore/libs/storage/init/actorsystem.h index 9f7dd198996..455f0a04916 100644 --- a/cloud/filestore/libs/storage/init/actorsystem.h +++ b/cloud/filestore/libs/storage/init/actorsystem.h @@ -34,7 +34,7 @@ struct TActorSystemArgs std::shared_ptr UserCounters; - NCloud::NStorage::ICgroupStatsFetcherPtr CgroupStatsFetcher; + NCloud::NStorage::IStatsFetcherPtr StatsFetcher; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/libs/storage/service/service.cpp b/cloud/filestore/libs/storage/service/service.cpp index d7d0321816a..5dbff3bc5d4 100644 --- a/cloud/filestore/libs/storage/service/service.cpp +++ b/cloud/filestore/libs/storage/service/service.cpp @@ -13,14 +13,14 @@ IActorPtr CreateStorageService( IRequestStatsRegistryPtr statsRegistry, IProfileLogPtr profileLog, ITraceSerializerPtr traceSerialzer, - NCloud::NStorage::ICgroupStatsFetcherPtr cgroupStatsFetcher) + NCloud::NStorage::IStatsFetcherPtr statsFetcher) { return std::make_unique( std::move(storageConfig), std::move(statsRegistry), std::move(profileLog), std::move(traceSerialzer), - std::move(cgroupStatsFetcher)); + std::move(statsFetcher)); } } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/service/service.h b/cloud/filestore/libs/storage/service/service.h index 7222307d268..1bb16f2f986 100644 --- a/cloud/filestore/libs/storage/service/service.h +++ b/cloud/filestore/libs/storage/service/service.h @@ -16,6 +16,6 @@ NActors::IActorPtr CreateStorageService( IRequestStatsRegistryPtr statsRegistry, IProfileLogPtr profileLog, ITraceSerializerPtr traceSerialzer, - NCloud::NStorage::ICgroupStatsFetcherPtr cgroupStatsFetcher); + NCloud::NStorage::IStatsFetcherPtr xtatsFetcher); } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/service/service_actor.cpp b/cloud/filestore/libs/storage/service/service_actor.cpp index b25c92887db..55c26722f7a 100644 --- a/cloud/filestore/libs/storage/service/service_actor.cpp +++ b/cloud/filestore/libs/storage/service/service_actor.cpp @@ -20,11 +20,11 @@ TStorageServiceActor::TStorageServiceActor( IRequestStatsRegistryPtr statsRegistry, IProfileLogPtr profileLog, ITraceSerializerPtr traceSerializer, - NCloud::NStorage::ICgroupStatsFetcherPtr cgroupStatsFetcher) + NCloud::NStorage::IStatsFetcherPtr statsFetcher) : StorageConfig{std::move(storageConfig)} , ProfileLog{std::move(profileLog)} , TraceSerializer{std::move(traceSerializer)} - , CgroupStatsFetcher(std::move(cgroupStatsFetcher)) + , StatsFetcher(std::move(statsFetcher)) , State{std::make_unique()} , StatsRegistry{std::move(statsRegistry)} {} diff --git a/cloud/filestore/libs/storage/service/service_actor.h b/cloud/filestore/libs/storage/service/service_actor.h index 15f81aff74f..8326547c209 100644 --- a/cloud/filestore/libs/storage/service/service_actor.h +++ b/cloud/filestore/libs/storage/service/service_actor.h @@ -36,7 +36,7 @@ class TStorageServiceActor final const TStorageConfigPtr StorageConfig; const IProfileLogPtr ProfileLog; const ITraceSerializerPtr TraceSerializer; - const NCloud::NStorage::ICgroupStatsFetcherPtr CgroupStatsFetcher; + const NCloud::NStorage::IStatsFetcherPtr StatsFetcher; std::unique_ptr State; ui64 ProxyCounter = 0; @@ -53,7 +53,7 @@ class TStorageServiceActor final IRequestStatsRegistryPtr statsRegistry, IProfileLogPtr profileLog, ITraceSerializerPtr traceSerializer, - NCloud::NStorage::ICgroupStatsFetcherPtr cgroupStatsFetcher); + NCloud::NStorage::IStatsFetcherPtr statsFetcher); ~TStorageServiceActor(); void Bootstrap(const NActors::TActorContext& ctx); diff --git a/cloud/filestore/libs/storage/service/service_actor_update_stats.cpp b/cloud/filestore/libs/storage/service/service_actor_update_stats.cpp index 9a1ad875e55..7ea8299b3ed 100644 --- a/cloud/filestore/libs/storage/service/service_actor_update_stats.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_update_stats.cpp @@ -32,28 +32,36 @@ void TStorageServiceActor::HandleUpdateStats( InFlightRequests.erase(it++); } } - if (CgroupStatsFetcher && CpuWait) { + if (StatsFetcher && CpuWait) { auto now = ctx.Now(); auto interval = (now - LastCpuWaitQuery).MicroSeconds(); - auto cpuWaitValue = CgroupStatsFetcher->GetCpuWait().MicroSeconds(); - auto cpuLack = CpuLackPercentsMultiplier * cpuWaitValue / interval; + auto cpuWait = StatsFetcher->GetCpuWait(); + if (!HasError(cpuWait)) { + auto cpuWaitValue = cpuWait.GetResult().MicroSeconds(); + auto cpuLack = CpuLackPercentsMultiplier * cpuWaitValue / interval; - LOG_DEBUG_S( - ctx, - TFileStoreComponents::SERVICE, - "CpuWait stats: lack = " << cpuLack << "; interval = " << interval - << "; wait = " << cpuWaitValue); - - *CpuWait = cpuLack; - LastCpuWaitQuery = now; - - if (cpuLack >= StorageConfig->GetCpuLackThreshold()) { - LOG_WARN_S( + LOG_DEBUG_S( + ctx, + TFileStoreComponents::SERVICE, + "CpuWait stats: lack = " << cpuLack << "; interval = " << interval + << "; wait = " << cpuWaitValue); + + *CpuWait = cpuLack; + LastCpuWaitQuery = now; + + if (cpuLack >= StorageConfig->GetCpuLackThreshold()) { + LOG_WARN_S( + ctx, + TFileStoreComponents::SERVICE, + "Cpu wait is " << cpuLack); + } + } else { + LOG_ERROR_S( ctx, TFileStoreComponents::SERVICE, - "Cpu wait is " << cpuLack); + "Failed to get Cpu wait " << cpuWait.GetError()); } } diff --git a/cloud/filestore/libs/storage/testlib/test_env.cpp b/cloud/filestore/libs/storage/testlib/test_env.cpp index ed17813544b..0ec99758748 100644 --- a/cloud/filestore/libs/storage/testlib/test_env.cpp +++ b/cloud/filestore/libs/storage/testlib/test_env.cpp @@ -223,7 +223,7 @@ ui32 TTestEnv::CreateNode(const TString& name) StatsRegistry, ProfileLog, TraceSerializer, - CreateCgroupStatsFetcherStub()); + CreateStatsFetcherStub()); auto indexServiceId = Runtime.Register( indexService.release(), nodeIdx, diff --git a/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher.cpp b/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher.cpp index 505b2ea46a5..047703da20d 100644 --- a/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher.cpp +++ b/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include @@ -13,6 +15,16 @@ #include #include +#include + +#include +#include +#include + +#include +#include +#include + namespace NCloud::NStorage { using namespace NMonitoring; @@ -22,7 +34,7 @@ namespace { //////////////////////////////////////////////////////////////////////////////// struct TCgroupStatsFetcher final - : public ICgroupStatsFetcher + : public IStatsFetcher { private: const TString ComponentName; @@ -75,17 +87,24 @@ struct TCgroupStatsFetcher final return; } - Last = GetCpuWait(); + + auto ret = GetCpuWait(); + if (!ret.HasError()) { + STORAGE_ERROR("Failed to get cpu stats: " << ret.GetError()); + } + else { + Last = ret.GetResult(); + } } void Stop() override { } - TDuration GetCpuWait() override + TResultOrError GetCpuWait() override { if (!CpuAcctWait.IsOpen()) { - return {}; + return MakeError(E_FAIL, "Failed to open stats file"); } try { @@ -95,9 +114,8 @@ struct TCgroupStatsFetcher final if (CpuAcctWait.GetLength() >= bufSize - 1) { ReportCpuWaitFatalError(); - STORAGE_ERROR(StatsFile << " is too large"); CpuAcctWait.Close(); - return {}; + return MakeError(E_FAIL, StatsFile + " is too large"); } char buf[bufSize]; @@ -110,13 +128,12 @@ struct TCgroupStatsFetcher final auto value = TDuration::MicroSeconds(FromString(buf) / 1000); if (value < Last) { - STORAGE_ERROR( - ReportCpuWaitCounterReadError( + auto errorMessage = ReportCpuWaitCounterReadError( TStringBuilder() << StatsFile << " : new value " << value << - " is less than previous " << Last)); + " is less than previous " << Last); Last = value; - return {}; + return MakeError(E_FAIL, std::move(errorMessage)); } auto retval = value - Last; Last = value; @@ -124,10 +141,12 @@ struct TCgroupStatsFetcher final return retval; } catch (...) { ReportCpuWaitFatalError(); - STORAGE_ERROR(BuildErrorMessageFromException()) + auto errorMessage = BuildErrorMessageFromException(); CpuAcctWait.Close(); - return {}; + return MakeError(E_FAIL, std::move(errorMessage)); } + + return MakeError(E_FAIL); } TString BuildErrorMessageFromException() @@ -158,8 +177,8 @@ struct TCgroupStatsFetcher final //////////////////////////////////////////////////////////////////////////////// -struct TCgroupStatsFetcherStub final - : public ICgroupStatsFetcher +struct TStatsFetcherStub final + : public IStatsFetcher { void Start() override { @@ -169,9 +188,123 @@ struct TCgroupStatsFetcherStub final { } - TDuration GetCpuWait() override + TResultOrError GetCpuWait() override { - return {}; + return TDuration{}; + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TKernelTaskDelayAcctStatsFetcher final: public IStatsFetcher +{ + using TNetlinkMessage = NCloud::NNetlink::TMessage; + using TNetlinkSocket = NCloud::NNetlink::TSocket; +private: + const TString ComponentName; + const ILoggingServicePtr Logging; + const IMonitoringServicePtr Monitoring; + TLog Log; + std::unique_ptr NetlinkSocket; + const TDuration NetlinkSocketTimeout = TDuration::Seconds(1); + +public: + TKernelTaskDelayAcctStatsFetcher( + TString componentName, + ILoggingServicePtr logging, + IMonitoringServicePtr monitoring) + : ComponentName(std::move(componentName)) + , Logging(std::move(logging)) + , Monitoring(std::move(monitoring)) + { + } + + ~TKernelTaskDelayAcctStatsFetcher() override { + Stop(); + } + + void Start() override + { + Log = Logging->CreateLog(ComponentName); + NetlinkSocket = std::make_unique(TASKSTATS_GENL_NAME); + } + + void Stop() override + { + NetlinkSocket.reset(); + } + + TResultOrError GetCpuWait() override + { + if (!NetlinkSocket) { + return MakeError(E_FAIL, "Invalid netlink socket"); + } + + try { + int mypid = getpid(); + TNetlinkMessage message( + NetlinkSocket->GetFamily(), + TASKSTATS_CMD_GET); + message.Put(TASKSTATS_CMD_ATTR_PID, mypid); + + auto cpuDelay = NThreading::NewPromise>(); + NetlinkSocket->SetCallback( + NL_CB_VALID, + [this, &cpuDelay](nl_msg* nlmsg) + { + auto delayNs = CpuDelayStatHandler(nlmsg); + if (delayNs == -1) { + cpuDelay.SetValue( + MakeError(E_FAIL, "Fail to parse netlink message")); + } else { + cpuDelay.SetValue( + TDuration::MilliSeconds(delayNs / 1000)); + } + return 0; + }); + NetlinkSocket->Send(message); + return cpuDelay.GetFuture().GetValue(NetlinkSocketTimeout); + } catch (...) { + auto errorMessage = BuildErrorMessageFromException(); + return MakeError(E_FAIL, errorMessage); + } + + return MakeError(E_FAIL); + } + + TString BuildErrorMessageFromException() + { + auto msg = TStringBuilder() << "IO error"; + msg << " with exception " << CurrentExceptionMessage(); + return msg; + } + + int CpuDelayStatHandler(nl_msg* nlmsg) + { + nlattr* nlattrs[TASKSTATS_TYPE_MAX + 1]; + if (auto rc = genlmsg_parse( + nlmsg_hdr(nlmsg), + 0, + nlattrs, + TASKSTATS_TYPE_MAX, + NULL); + rc < 0) + { + std::cerr << "error parsing msg" << std::endl; + return -1; + } + + if (auto nlattr = nlattrs[TASKSTATS_TYPE_AGGR_PID]; nlattr != nullptr) + { + const auto pdata = + reinterpret_cast(nla_data(nlattr)); + int rem = 0; + auto stats = + reinterpret_cast(nla_data(nla_next(pdata, &rem))); + return stats->cpu_delay_total; + } + std::cerr << "unknown attribute format received" << std::endl; + return -1; } }; @@ -179,7 +312,7 @@ struct TCgroupStatsFetcherStub final //////////////////////////////////////////////////////////////////////////////// -ICgroupStatsFetcherPtr CreateCgroupStatsFetcher( +IStatsFetcherPtr CreateCgroupStatsFetcher( TString componentName, ILoggingServicePtr logging, IMonitoringServicePtr monitoring, @@ -194,9 +327,20 @@ ICgroupStatsFetcherPtr CreateCgroupStatsFetcher( std::move(settings)); } -ICgroupStatsFetcherPtr CreateCgroupStatsFetcherStub() +IStatsFetcherPtr CreateKernelTaskDelayAcctStatsFetcher( + TString componentName, + ILoggingServicePtr logging, + IMonitoringServicePtr monitoring) +{ + return std::make_shared( + std::move(componentName), + std::move(logging), + std::move(monitoring)); +} + +IStatsFetcherPtr CreateStatsFetcherStub() { - return std::make_shared(); + return std::make_shared(); } TString BuildCpuWaitStatsFilename(const TString& serviceName) diff --git a/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher.h b/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher.h index 9cd559ef1a1..17cb04e5857 100644 --- a/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher.h +++ b/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher.h @@ -2,6 +2,7 @@ #include "public.h" +#include #include #include @@ -16,15 +17,15 @@ namespace NCloud::NStorage { //////////////////////////////////////////////////////////////////////////////// -struct ICgroupStatsFetcher +struct IStatsFetcher : public IStartable { - virtual ~ICgroupStatsFetcher() = default; + virtual ~IStatsFetcher() = default; - virtual TDuration GetCpuWait() = 0; + virtual TResultOrError GetCpuWait() = 0; }; -using ICgroupStatsFetcherPtr = std::shared_ptr; +using IStatsFetcherPtr = std::shared_ptr; //////////////////////////////////////////////////////////////////////////////// @@ -35,14 +36,19 @@ struct TCgroupStatsFetcherMonitoringSettings TString CounterName; }; -ICgroupStatsFetcherPtr CreateCgroupStatsFetcher( +IStatsFetcherPtr CreateCgroupStatsFetcher( TString componentName, ILoggingServicePtr logging, IMonitoringServicePtr monitoring, TString statsFile, TCgroupStatsFetcherMonitoringSettings settings); -ICgroupStatsFetcherPtr CreateCgroupStatsFetcherStub(); +IStatsFetcherPtr CreateKernelTaskDelayAcctStatsFetcher( + TString componentName, + ILoggingServicePtr logging, + IMonitoringServicePtr monitoring); + +IStatsFetcherPtr CreateStatsFetcherStub(); TString BuildCpuWaitStatsFilename(const TString& serviceName); diff --git a/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher_ut.cpp b/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher_ut.cpp index 31f6861a0ac..635aa3d4af7 100644 --- a/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher_ut.cpp +++ b/cloud/storage/core/libs/diagnostics/cgroup_stats_fetcher_ut.cpp @@ -66,14 +66,26 @@ Y_UNIT_TEST_SUITE(TCGroupStatFetcherTest) }); fetcher->Start(); - UNIT_ASSERT_VALUES_EQUAL(TDuration(), fetcher->GetCpuWait()); + auto cpuWait = fetcher->GetCpuWait(); + UNIT_ASSERT_C(!HasError(cpuWait), cpuWait.GetError()); + UNIT_ASSERT_VALUES_EQUAL( + TDuration::MicroSeconds(0), + cpuWait.GetResult()); UpdateCGroupWaitDuration(statsFile, TDuration::MicroSeconds(20)); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(10), fetcher->GetCpuWait()); + cpuWait = fetcher->GetCpuWait(); + UNIT_ASSERT_C(!HasError(cpuWait), cpuWait.GetError()); + UNIT_ASSERT_VALUES_EQUAL( + TDuration::MicroSeconds(10), + cpuWait.GetResult()); fetcher->Stop(); - UNIT_ASSERT_VALUES_EQUAL(0, serverGroup->GetCounter("AppCriticalEvents/CpuWaitCounterReadError", true)->Val()); + UNIT_ASSERT_VALUES_EQUAL( + 0, + serverGroup + ->GetCounter("AppCriticalEvents/CpuWaitCounterReadError", true) + ->Val()); } Y_UNIT_TEST(ShouldReportErrorIfFileIsMissing) @@ -100,8 +112,10 @@ Y_UNIT_TEST_SUITE(TCGroupStatFetcherTest) UNIT_ASSERT_VALUES_EQUAL(1, failCounter->Val()); - UNIT_ASSERT_VALUES_EQUAL(TDuration(), fetcher->GetCpuWait()); - UNIT_ASSERT_VALUES_EQUAL(TDuration(), fetcher->GetCpuWait()); + auto cpuWait = fetcher->GetCpuWait(); + UNIT_ASSERT_C(HasError(cpuWait), cpuWait.GetError()); + cpuWait = fetcher->GetCpuWait(); + UNIT_ASSERT_C(HasError(cpuWait), cpuWait.GetError()); fetcher->Stop(); } @@ -127,12 +141,25 @@ Y_UNIT_TEST_SUITE(TCGroupStatFetcherTest) fetcher->Start(); UpdateCGroupWaitDuration(statsFile, TDuration::MicroSeconds(80)); - UNIT_ASSERT_VALUES_EQUAL(TDuration(), fetcher->GetCpuWait()); - UNIT_ASSERT_VALUES_EQUAL(1, serverGroup->GetCounter("AppCriticalEvents/CpuWaitCounterReadError", true)->Val()); + auto cpuWait = fetcher->GetCpuWait(); + UNIT_ASSERT_C(HasError(cpuWait), cpuWait.GetError()); + UNIT_ASSERT_VALUES_EQUAL( + 1, + serverGroup + ->GetCounter("AppCriticalEvents/CpuWaitCounterReadError", true) + ->Val()); UpdateCGroupWaitDuration(statsFile, TDuration::MicroSeconds(100)); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(20), fetcher->GetCpuWait()); - UNIT_ASSERT_VALUES_EQUAL(1, serverGroup->GetCounter("AppCriticalEvents/CpuWaitCounterReadError", true)->Val()); + cpuWait = fetcher->GetCpuWait(); + UNIT_ASSERT_C(!HasError(cpuWait), cpuWait.GetError()); + UNIT_ASSERT_VALUES_EQUAL( + TDuration::MicroSeconds(20), + cpuWait.GetResult()); + UNIT_ASSERT_VALUES_EQUAL( + 1, + serverGroup + ->GetCounter("AppCriticalEvents/CpuWaitCounterReadError", true) + ->Val()); fetcher->Stop(); } diff --git a/cloud/storage/core/libs/diagnostics/netlink_ut/bin/ya.make b/cloud/storage/core/libs/diagnostics/netlink_ut/bin/ya.make new file mode 100644 index 00000000000..3521cb15e02 --- /dev/null +++ b/cloud/storage/core/libs/diagnostics/netlink_ut/bin/ya.make @@ -0,0 +1,9 @@ +UNITTEST_FOR(cloud/storage/core/libs/diagnostics) + +IF (OS_LINUX) + SRCS( + stats_fetcher_ut.cpp + ) +ENDIF() + +END() \ No newline at end of file diff --git a/cloud/storage/core/libs/diagnostics/netlink_ut/test.py b/cloud/storage/core/libs/diagnostics/netlink_ut/test.py new file mode 100644 index 00000000000..9dc2483388f --- /dev/null +++ b/cloud/storage/core/libs/diagnostics/netlink_ut/test.py @@ -0,0 +1,9 @@ +import yatest.common as common + +tests_bin = "storage-core-libs-diagnostics-netlink_ut-bin" +tests_bin_path = "cloud/storage/core/libs/diagnostics/netlink_ut/bin/" + tests_bin + + +def test_qemu_netlink_ut(): + test_tool = common.binary_path(tests_bin_path) + common.execute(test_tool) diff --git a/cloud/storage/core/libs/diagnostics/netlink_ut/ya.make b/cloud/storage/core/libs/diagnostics/netlink_ut/ya.make new file mode 100644 index 00000000000..69009cbd828 --- /dev/null +++ b/cloud/storage/core/libs/diagnostics/netlink_ut/ya.make @@ -0,0 +1,17 @@ +PY3TEST() + +INCLUDE(${ARCADIA_ROOT}/cloud/storage/core/tests/recipes/medium.inc) +SPLIT_FACTOR(1) + +DEPENDS( + cloud/storage/core/libs/endpoints/keyring/ut/bin + cloud/storage/core/libs/diagnostics/netlink_ut/bin +) + +TEST_SRCS( + test.py +) + +INCLUDE(${ARCADIA_ROOT}/cloud/storage/core/tests/recipes/qemu.inc) + +END() diff --git a/cloud/storage/core/libs/diagnostics/public.h b/cloud/storage/core/libs/diagnostics/public.h index 09a483d193f..91b0af14d31 100644 --- a/cloud/storage/core/libs/diagnostics/public.h +++ b/cloud/storage/core/libs/diagnostics/public.h @@ -84,8 +84,8 @@ namespace NStorage { //////////////////////////////////////////////////////////////////////////////// -struct ICgroupStatsFetcher; -using ICgroupStatsFetcherPtr = std::shared_ptr; +struct IStatsFetcher; +using IStatsFetcherPtr = std::shared_ptr; } // namespace NStorage diff --git a/cloud/storage/core/libs/diagnostics/stats_fetcher_ut.cpp b/cloud/storage/core/libs/diagnostics/stats_fetcher_ut.cpp new file mode 100644 index 00000000000..ab63c993287 --- /dev/null +++ b/cloud/storage/core/libs/diagnostics/stats_fetcher_ut.cpp @@ -0,0 +1,41 @@ +#include "cgroup_stats_fetcher.h" + +#include "critical_events.h" + +#include +#include + +#include +#include + +#include + +namespace NCloud::NStorage { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +const TString ComponentName = "STORAGE_CGROUPS"; + +}; //namespace + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(StatFetcherTest) +{ + Y_UNIT_TEST(ShouldGetCpuDelay) + { + auto monitoring = CreateMonitoringServiceStub(); + auto fetcher = CreateKernelTaskDelayAcctStatsFetcher( + ComponentName, + CreateLoggingService("console"), + monitoring); + fetcher->Start(); + auto cpuWait = fetcher->GetCpuWait(); + UNIT_ASSERT_C(!HasError(cpuWait), cpuWait.GetError()); + fetcher->Stop(); + } +} + +} // namespace NCloud::NStorage diff --git a/cloud/storage/core/libs/diagnostics/ya.make b/cloud/storage/core/libs/diagnostics/ya.make index 78876ea5f3a..02b5f394989 100644 --- a/cloud/storage/core/libs/diagnostics/ya.make +++ b/cloud/storage/core/libs/diagnostics/ya.make @@ -25,10 +25,13 @@ SRCS( PEERDIR( cloud/storage/core/libs/common + cloud/storage/core/libs/netlink cloud/storage/core/protos library/cpp/lwtrace/mon + contrib/restricted/libnl/lib/nl-3 + contrib/restricted/libnl/lib/nl-genl-3 contrib/ydb/library/actors/prof library/cpp/containers/ring_buffer library/cpp/deprecated/atomic @@ -46,4 +49,5 @@ PEERDIR( END() +RECURSE_FOR_TESTS(netlink_ut) RECURSE_FOR_TESTS(ut) diff --git a/cloud/storage/core/libs/netlink/message.cpp b/cloud/storage/core/libs/netlink/message.cpp new file mode 100644 index 00000000000..41065acd745 --- /dev/null +++ b/cloud/storage/core/libs/netlink/message.cpp @@ -0,0 +1,55 @@ +#include "message.h" + +namespace NCloud::NNetlink { + +TNestedAttribute::TNestedAttribute(nl_msg* message, int attribute) + : Message(message) +{ + Attribute = nla_nest_start(message, attribute); + if (!Attribute) { + throw TServiceError(E_FAIL) << "unable to nest attribute"; + } +} + +TNestedAttribute::~TNestedAttribute() +{ + nla_nest_end(Message, Attribute); +} + +TMessage::TMessage(int family, int command) +{ + Message = nlmsg_alloc(); + if (Message == nullptr) { + throw TServiceError(E_FAIL) << "unable to allocate message"; + } + genlmsg_put( + Message, + NL_AUTO_PORT, + NL_AUTO_SEQ, + family, + 0, // hdrlen + 0, // flags + command, + 1); // version +} + +TMessage::~TMessage() +{ + nlmsg_free(Message); +} + +TNestedAttribute TMessage::Nest(int attribute) +{ + return TNestedAttribute(Message, attribute); +} + +void TMessage::Put(int attribute, void* data, size_t size) +{ + if (int err = nla_put(Message, attribute, size, data)) { + throw TServiceError(E_FAIL) + << "unable to put attribute " << attribute << ": " + << nl_geterror(err); + } +} + +} // namespace NCloud::NNetlink diff --git a/cloud/storage/core/libs/netlink/message.h b/cloud/storage/core/libs/netlink/message.h new file mode 100644 index 00000000000..a6573dfb99c --- /dev/null +++ b/cloud/storage/core/libs/netlink/message.h @@ -0,0 +1,48 @@ +#include + +#include +#include + +namespace NCloud::NNetlink { + +//////////////////////////////////////////////////////////////////////////////// + +class TNestedAttribute +{ +private: + nl_msg* Message; + nlattr* Attribute; + +public: + TNestedAttribute(nl_msg* message, int attribute); + ~TNestedAttribute(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TMessage +{ +private: + nl_msg* Message; + +public: + TMessage(int family, int command); + ~TMessage(); + + operator nl_msg*() const + { + return Message; + } + + TNestedAttribute Nest(int attribute); + + void Put(int attribute, void* data, size_t size); + + template + void Put(int attribute, T data) + { + Put(attribute, &data, sizeof(T)); + } +}; + +} // namespace NCloud::NNetlink diff --git a/cloud/storage/core/libs/netlink/socket.cpp b/cloud/storage/core/libs/netlink/socket.cpp new file mode 100644 index 00000000000..027bc1686b0 --- /dev/null +++ b/cloud/storage/core/libs/netlink/socket.cpp @@ -0,0 +1,63 @@ +#include "socket.h" + +namespace NCloud::NNetlink { + +TSocket::TSocket(TString family) + : Socket(nl_socket_alloc(), [](auto* socket) { nl_socket_free(socket); }) +{ + if (!Socket) { + throw TServiceError(E_FAIL) << "unable to allocate netlink socket"; + } + + if (int err = genl_connect(Socket.get())) { + throw TServiceError(E_FAIL) + << "unable to connect to generic netlink socket: " + << nl_geterror(err); + } + + Family = genl_ctrl_resolve(Socket.get(), family.c_str()); + + if (Family < 0) { + throw TServiceError(E_FAIL) + << "unable to resolve netlink family: " << nl_geterror(Family); + } +} + +void TSocket::SetCallback(nl_cb_type type, TResponseHandler func) +{ + auto arg = std::make_unique(std::move(func)); + + if (int err = nl_socket_modify_cb( + Socket.get(), + type, + NL_CB_CUSTOM, + TSocket::ResponseHandler, + arg.get())) + { + throw TServiceError(E_FAIL) + << "unable to set socket callback: " << nl_geterror(err); + } + arg.release(); +} + +void TSocket::Send(nl_msg* message) +{ + if (int err = nl_send_auto(Socket.get(), message); err < 0) { + throw TServiceError(E_FAIL) << "send error: " << nl_geterror(err); + } + if (int err = nl_wait_for_ack(Socket.get())) { + // this is either recv error, or an actual error message received + // from the kernel + throw TServiceError(E_FAIL) << "recv error: " << nl_geterror(err); + } +} + +int TSocket::ResponseHandler(nl_msg* msg, void* arg) +{ + auto func = std::unique_ptr( + static_cast(arg)); + + return (*func)(msg); +} + +} // namespace NCloud::NNetlink diff --git a/cloud/storage/core/libs/netlink/socket.h b/cloud/storage/core/libs/netlink/socket.h new file mode 100644 index 00000000000..42b73b889bb --- /dev/null +++ b/cloud/storage/core/libs/netlink/socket.h @@ -0,0 +1,39 @@ +#pragma once + +#include "message.h" + +#include + +#include + +#include +#include +#include + +namespace NCloud::NNetlink { + +using TResponseHandler = std::function; + +class TSocket +{ +private: + std::unique_ptr Socket; + int Family; + +public: + explicit TSocket(TString family); + ~TSocket() = default; + + [[nodiscard]] int GetFamily() const + { + return Family; + } + + void SetCallback(nl_cb_type type, TResponseHandler func); + + static int ResponseHandler(nl_msg* msg, void* arg); + + void Send(nl_msg* message); +}; + +} // namespace NCloud::NNetlink diff --git a/cloud/storage/core/libs/netlink/ya.make b/cloud/storage/core/libs/netlink/ya.make new file mode 100644 index 00000000000..16661590b8d --- /dev/null +++ b/cloud/storage/core/libs/netlink/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +LICENSE_RESTRICTION_EXCEPTIONS( + contrib/restricted/libnl/lib/nl-3 + contrib/restricted/libnl/lib/nl-genl-3 +) + +SRCS( + message.cpp + socket.cpp +) + +PEERDIR( + cloud/storage/core/libs/common + contrib/restricted/libnl/lib/nl-3 + contrib/restricted/libnl/lib/nl-genl-3 +) + +END() diff --git a/cloud/storage/core/libs/ya.make b/cloud/storage/core/libs/ya.make index 2056c321172..65a50cbab87 100644 --- a/cloud/storage/core/libs/ya.make +++ b/cloud/storage/core/libs/ya.make @@ -13,6 +13,7 @@ RECURSE( hive_proxy http kikimr + netlink ss_proxy tablet throttling diff --git a/cloud/storage/core/tools/analytics/cpu-wait-monitor/main.cpp b/cloud/storage/core/tools/analytics/cpu-wait-monitor/main.cpp index f248e10348c..31854289e43 100644 --- a/cloud/storage/core/tools/analytics/cpu-wait-monitor/main.cpp +++ b/cloud/storage/core/tools/analytics/cpu-wait-monitor/main.cpp @@ -80,10 +80,15 @@ int main(int argc, const char** argv) while (!options.SampleCount || numSamples--) { Sleep(pollInterval); - auto waitTime = 100 * statsFetcher->GetCpuWait().MicroSeconds(); - auto interval = pollInterval.MicroSeconds(); - - Cout << (waitTime / interval) << Endl; + auto cpuWait = statsFetcher->GetCpuWait(); + if (!HasError(cpuWait)) { + auto waitTime = 100 * cpuWait.GetResult().MicroSeconds(); + auto interval = pollInterval.MicroSeconds(); + Cout << (waitTime / interval) << Endl; + } + else { + Cout << cpuWait.GetError() << Endl; + } } return 0;