[NBS] make new volume counters to fix UsedQuota strange values

cloud/blockstore/config/storage.proto

@@ -1074,4 +1074,7 @@ message TStorageServiceConfig
 
     // Enabling direct copying of data between disk agents.
     optional bool UseDirectCopyRange = 394;
+
+    // Enabling UsedQuota calculation as UsedIoQuota + UsedBandwidthQuota
+    optional bool CalculateUsedQuotaFromOtherMetrics = 395;
 }

cloud/blockstore/libs/storage/core/config.cpp

@@ -491,6 +491,7 @@ TDuration MSeconds(ui32 value)
     xxx(VolumeProxyCacheRetryDuration,             TDuration, Seconds(15)     )\
                                                                                \
     xxx(UseDirectCopyRange,                             bool,      false         )\
+    xxx(CalculateUsedQuotaFromOtherMetrics,             bool,      false         )\
     xxx(MaxShadowDiskFillBandwidth,                     ui32,      512           )\
     xxx(MaxShadowDiskFillIoDepth,                       ui32,      1             )\
     xxx(BackgroundOperationsTotalBandwidth,             ui32,      1024          )\

cloud/blockstore/libs/storage/core/config.h

@@ -572,6 +572,7 @@ class TStorageConfig
     TString GetCachedDiskAgentSessionsPath() const;
 
     bool GetUseDirectCopyRange() const;
+    bool GetCalculateUsedQuotaFromOtherMetrics() const;
     ui32 GetMaxShadowDiskFillBandwidth() const;
     ui32 GetMaxShadowDiskFillIoDepth() const;
     ui32 GetBackgroundOperationsTotalBandwidth() const;

cloud/blockstore/libs/storage/core/disk_counters.h

@@ -623,12 +623,22 @@ struct TVolumeSelfCumulativeCounters
         EPublishingPolicy::All,
         TCumulativeCounter::ECounterType::Generic,
         ECounterExpirationPolicy::Permanent};
+    TCounter UsedIoQuota{
+        EPublishingPolicy::All,
+        TCumulativeCounter::ECounterType::Generic,
+        ECounterExpirationPolicy::Permanent};
+    TCounter UsedBandwidthQuota{
+        EPublishingPolicy::All,
+        TCumulativeCounter::ECounterType::Generic,
+        ECounterExpirationPolicy::Permanent};
 
     static constexpr TMeta AllCounters[] = {
         MakeMeta<&TVolumeSelfCumulativeCounters::ThrottlerRejectedRequests>(),
         MakeMeta<&TVolumeSelfCumulativeCounters::ThrottlerPostponedRequests>(),
         MakeMeta<&TVolumeSelfCumulativeCounters::ThrottlerSkippedRequests>(),
         MakeMeta<&TVolumeSelfCumulativeCounters::UsedQuota>(),
+        MakeMeta<&TVolumeSelfCumulativeCounters::UsedIoQuota>(),
+        MakeMeta<&TVolumeSelfCumulativeCounters::UsedBandwidthQuota>(),
     };
 };
 static_assert(

cloud/blockstore/libs/storage/volume/model/volume_throttling_policy.cpp

@@ -80,6 +80,9 @@ struct TVolumeThrottlingPolicy::TImpl
     double WriteCostMultiplier = 1;
     ui32 PostponedWeight = 0;
 
+    double UsedIoOperationsBudget = 0;
+    double UsedBandwidthQuota = 0;
+
     TImpl(
             const NProto::TVolumePerformanceProfile& config,
             const ui32 policyVersion,
@@ -236,7 +239,7 @@ struct TVolumeThrottlingPolicy::TImpl
             return TDuration::Zero();
         }
 
-        ui32 bandwidthUpdate = requestInfo.ByteCount;
+        const ui32 bandwidthUpdate = requestInfo.ByteCount;
         double m = static_cast<EOpType>(requestInfo.OpType) == EOpType::Read
             ? 1.0
             : WriteCostMultiplier;
@@ -245,16 +248,25 @@ struct TVolumeThrottlingPolicy::TImpl
             MaxBandwidth(static_cast<EOpType>(requestInfo.OpType));
         const auto maxIops = MaxIops(static_cast<EOpType>(requestInfo.OpType));
 
+        const auto recalculatedMaxIops =
+            CalculateThrottlerC1(maxIops, maxBandwidth);
+        const auto recalculatedMaxBandwidth =
+            CalculateThrottlerC2(maxIops, maxBandwidth);
         auto d = Bucket.Register(
             ts,
             m * CostPerIO(
-                CalculateThrottlerC1(maxIops, maxBandwidth),
-                CalculateThrottlerC2(maxIops, maxBandwidth),
-                bandwidthUpdate
-            )
-        );
+                    recalculatedMaxIops,
+                    recalculatedMaxBandwidth,
+                    bandwidthUpdate));
 
         if (!d.GetValue()) {
+            // 0 is special value which disables throttling by byteCount
+            if (maxBandwidth) {
+                UsedBandwidthQuota +=
+                    m * (static_cast<double>(bandwidthUpdate) /
+                         static_cast<double>(recalculatedMaxBandwidth));
+            }
+            UsedIoOperationsBudget += m * (1.0 / recalculatedMaxIops);
             return TDuration::Zero();
         }
 
@@ -283,6 +295,20 @@ struct TVolumeThrottlingPolicy::TImpl
     {
         return Bucket.CalculateCurrentSpentBudgetShare(ts);
     }
+
+    double TakeUsedIoQuota()
+    {
+        auto res = UsedIoOperationsBudget;
+        UsedIoOperationsBudget = 0;
+        return res;
+    }
+
+    double TakeUsedBandwidthQuota()
+    {
+        auto res = UsedBandwidthQuota;
+        UsedBandwidthQuota = 0;
+        return res;
+    }
 };
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -390,6 +416,16 @@ double TVolumeThrottlingPolicy::CalculateCurrentSpentBudgetShare(TInstant ts) co
     return Impl->CalculateCurrentSpentBudgetShare(ts);
 }
 
+double TVolumeThrottlingPolicy::TakeUsedIoQuota()
+{
+    return Impl->TakeUsedIoQuota();
+}
+
+double TVolumeThrottlingPolicy::TakeUsedBandwidthQuota()
+{
+    return Impl->TakeUsedBandwidthQuota();
+}
+
 const TBackpressureReport& TVolumeThrottlingPolicy::GetCurrentBackpressure() const
 {
     return Impl->CurrentBackpressure;

cloud/blockstore/libs/storage/volume/model/volume_throttling_policy.h

@@ -99,6 +99,8 @@ class TVolumeThrottlingPolicy final
     TDuration GetCurrentBoostBudget() const;
     ui32 CalculatePostponedWeight() const;
     double CalculateCurrentSpentBudgetShare(TInstant ts) const;
+    [[nodiscard]] double TakeUsedIoQuota();
+    [[nodiscard]] double TakeUsedBandwidthQuota();
     const TBackpressureReport& GetCurrentBackpressure() const;
     const NProto::TVolumePerformanceProfile& GetConfig() const;
 

cloud/blockstore/libs/storage/volume/model/volume_throttling_policy_ut.cpp

@@ -1,4 +1,5 @@
 #include "volume_throttling_policy.h"
+#include "cloud/storage/core/libs/throttling/helpers.h"
 
 #include <cloud/blockstore/libs/storage/protos/part.pb.h>
 
@@ -643,6 +644,99 @@ Y_UNIT_TEST_SUITE(TVolumeThrottlingPolicyTest)
         DO_TEST(tp, 4'379'770, 10'000, 1_MB, static_cast<ui32>(EOpType::Write));
     }
 
+    Y_UNIT_TEST(CollectThrotlingDiagnostics)
+    {
+        const ui64 maxBandwidth = 2_MB;
+        const ui64 maxIops = 4;
+
+        const auto config = MakeSimpleConfig(
+            maxBandwidth,
+            maxIops,
+            100,   // burstPercentage
+            0,     // boostTime
+            0,     // boostRefillTime
+            0,     // boostPercentage
+            1_GB   // maxPostponedWeight
+        );
+        TVolumeThrottlingPolicy tp(
+            config,
+            TThrottlerConfig(
+                TDuration::Seconds(1),        // maxDelay
+                Max<ui32>(),                  // maxWriteCostMultiplier
+                1,                            // defaultPostponedRequestWeight
+                CalculateBoostTime(config),   // initialBoostBudget
+                false                         // useDiskSpaceScore
+                ));
+
+        const auto byteCount = 256_KB;
+        const auto ioOperation = 2;
+
+        for (auto i = 0; i < ioOperation; ++i) {
+            DO_TEST(tp, 0, 0, byteCount, static_cast<ui32>(EOpType::Read));
+        }
+
+        auto recalculatedMaxIops = CalculateThrottlerC1(maxIops, maxBandwidth);
+        auto recalculatedMaxBandwidth =
+            CalculateThrottlerC2(maxIops, maxBandwidth);
+
+        UNIT_ASSERT_DOUBLES_EQUAL(
+            tp.TakeUsedBandwidthQuota(),
+            static_cast<double>(ioOperation * byteCount) /
+                static_cast<double>(recalculatedMaxBandwidth),
+            1e-6);
+
+        UNIT_ASSERT_DOUBLES_EQUAL(tp.TakeUsedBandwidthQuota(), 0, 1e-6);
+
+        UNIT_ASSERT_DOUBLES_EQUAL(
+            tp.TakeUsedIoQuota(),
+            static_cast<double>(ioOperation) /
+                static_cast<double>(recalculatedMaxIops),
+            1e-6);
+
+        UNIT_ASSERT_DOUBLES_EQUAL(tp.TakeUsedIoQuota(), 0, 1e-6);
+    }
+
+    Y_UNIT_TEST(UsedBandwidthQuotaZeroWithoutBytesThrotling)
+    {
+        const ui64 maxIops = 4;
+
+        const auto config = MakeSimpleConfig(
+            0,
+            maxIops,
+            100,   // burstPercentage
+            0,     // boostTime
+            0,     // boostRefillTime
+            0,     // boostPercentage
+            1_GB   // maxPostponedWeight
+        );
+        TVolumeThrottlingPolicy tp(
+            config,
+            TThrottlerConfig(
+                TDuration::Seconds(1),        // maxDelay
+                Max<ui32>(),                  // maxWriteCostMultiplier
+                1,                            // defaultPostponedRequestWeight
+                CalculateBoostTime(config),   // initialBoostBudget
+                false                         // useDiskSpaceScore
+                ));
+
+        const auto byteCount = 256_KB;
+        const auto ioOperation = 2;
+
+        for (auto i = 0; i < ioOperation; ++i) {
+            DO_TEST(tp, 0, 0, byteCount, static_cast<ui32>(EOpType::Read));
+        }
+
+        UNIT_ASSERT_DOUBLES_EQUAL(
+            tp.TakeUsedBandwidthQuota(),
+            0,
+            1e-6);
+
+        UNIT_ASSERT_DOUBLES_EQUAL(
+            tp.TakeUsedIoQuota(),
+            static_cast<double>(ioOperation) / static_cast<double>(maxIops),
+            1e-6);
+    }
+
 #undef DO_TEST
 
     Y_UNIT_TEST(MergeBackpressureReports)

cloud/blockstore/libs/storage/volume/volume_actor.cpp

@@ -215,8 +215,26 @@ void TVolumeActor::UpdateLeakyBucketCounters(const TActorContext& ctx)
 
     auto& simple = VolumeSelfCounters->Simple;
     auto& cumulative = VolumeSelfCounters->Cumulative;
-    const auto& tp = State->GetThrottlingPolicy();
-    ui64 currentRate = tp.CalculateCurrentSpentBudgetShare(ctx.Now()) * 100;
+    auto& tp = State->AccessThrottlingPolicy();
+
+    auto usedIoQuota = tp.TakeUsedIoQuota();
+    auto usedBandwidthQuota = tp.TakeUsedBandwidthQuota();
+
+    cumulative.UsedIoQuota.Increment(static_cast<ui64>(usedIoQuota * 100.0));
+    cumulative.UsedBandwidthQuota.Increment(
+        static_cast<ui64>(usedBandwidthQuota * 100.0));
+
+    Config->GetCalculateUsedQuotaFromOtherMetrics();
+
+    auto currentRate = static_cast<ui64>(Min(
+        [&]()
+        {
+            if (Config->GetCalculateUsedQuotaFromOtherMetrics()) {
+                return (usedIoQuota + usedBandwidthQuota) * 100.0;
+            }
+            return tp.CalculateCurrentSpentBudgetShare(ctx.Now()) * 100.0;
+        }(),
+        100.0));
     simple.MaxUsedQuota.Set(Max(simple.MaxUsedQuota.Value, currentRate));
     cumulative.UsedQuota.Increment(currentRate);
 }

cloud/blockstore/libs/throttling/throttler.cpp

@@ -530,8 +530,10 @@ class TThrottler final
     {
         ui64 quota = 0;
         with_lock (ThrottlerLock) {
-            quota = static_cast<ui64>(ceil(ThrottlerPolicy
-                ->CalculateCurrentSpentBudgetShare(Timer->Now()) * 100.0));
+            quota = static_cast<ui64>(
+                ThrottlerPolicy->CalculateCurrentSpentBudgetShare(
+                    Timer->Now()) *
+                100.0);
         }
         ThrottlerMetrics->UpdateUsedQuota(quota);
         ThrottlerMetrics->UpdateMaxUsedQuota();