From 3aa34cfbe6f525e3f0676b98981a53f0815e04dc Mon Sep 17 00:00:00 2001 From: Evgeny Budilovsky Date: Thu, 21 Mar 2024 17:22:06 +0200 Subject: [PATCH] NBSNEBIUS-148: implement throttling for fast data path --- .../endpoints_vhost/external_vhost_server.cpp | 11 ++ .../external_vhost_server_ut.cpp | 15 +- cloud/blockstore/vhost-server/backend.h | 1 + cloud/blockstore/vhost-server/backend_aio.cpp | 5 + .../blockstore/vhost-server/backend_null.cpp | 5 + .../blockstore/vhost-server/backend_rdma.cpp | 90 ++++++++--- cloud/blockstore/vhost-server/options.cpp | 18 +++ cloud/blockstore/vhost-server/options.h | 5 + cloud/blockstore/vhost-server/options_ut.cpp | 50 ++++++ cloud/blockstore/vhost-server/public.h | 3 + cloud/blockstore/vhost-server/server.cpp | 2 + cloud/blockstore/vhost-server/throttler.cpp | 138 +++++++++++++++++ cloud/blockstore/vhost-server/throttler.h | 29 ++++ .../blockstore/vhost-server/throttler_ut.cpp | 145 ++++++++++++++++++ cloud/blockstore/vhost-server/ut/ya.make | 4 + cloud/blockstore/vhost-server/ya.make | 1 + 16 files changed, 501 insertions(+), 21 deletions(-) create mode 100644 cloud/blockstore/vhost-server/throttler.cpp create mode 100644 cloud/blockstore/vhost-server/throttler.h create mode 100644 cloud/blockstore/vhost-server/throttler_ut.cpp diff --git a/cloud/blockstore/libs/endpoints_vhost/external_vhost_server.cpp b/cloud/blockstore/libs/endpoints_vhost/external_vhost_server.cpp index 7ad704b7c61..db3f6fe9309 100644 --- a/cloud/blockstore/libs/endpoints_vhost/external_vhost_server.cpp +++ b/cloud/blockstore/libs/endpoints_vhost/external_vhost_server.cpp @@ -880,6 +880,17 @@ class TExternalVhostEndpointListener final args.emplace_back("--read-only"); } + if (epType == EEndpointType::Rdma) { + const auto& profile = volume.GetPerformanceProfile(); + args.insert(args.end(), { + "--perf-profile", TStringBuilder() + << profile.GetMaxReadBandwidth() << ":" + << profile.GetMaxReadIops() << ":" + << profile.GetMaxWriteBandwidth() << ":" + << profile.GetMaxWriteIops() + }); + } + TVector cgroups( request.GetClientCGroups().begin(), request.GetClientCGroups().end() diff --git a/cloud/blockstore/libs/endpoints_vhost/external_vhost_server_ut.cpp b/cloud/blockstore/libs/endpoints_vhost/external_vhost_server_ut.cpp index ac7100182e3..5c6acde15de 100644 --- a/cloud/blockstore/libs/endpoints_vhost/external_vhost_server_ut.cpp +++ b/cloud/blockstore/libs/endpoints_vhost/external_vhost_server_ut.cpp @@ -217,6 +217,12 @@ struct TFixture volume.SetStorageMediaKind(NProto::STORAGE_MEDIA_SSD_NONREPLICATED); volume.SetIsFastPathEnabled(true); + auto* profile = volume.MutablePerformanceProfile(); + profile->SetMaxReadBandwidth(1111); + profile->SetMaxReadIops(2222); + profile->SetMaxWriteBandwidth(3333); + profile->SetMaxWriteIops(4444); + { auto* device = volume.AddDevices(); device->SetDeviceName("/dev/disk/by-path/pci-0000:00:16.0-sas-phy2-lun-0"); @@ -477,10 +483,11 @@ Y_UNIT_TEST_SUITE(TExternalEndpointTest) --device ... 2 --device ... 2 --read-only 1 - 19 + --perf-profile 2 + 21 */ - UNIT_ASSERT_VALUES_EQUAL(19, create->CmdArgs.size()); + UNIT_ASSERT_VALUES_EQUAL(21, create->CmdArgs.size()); UNIT_ASSERT_VALUES_EQUAL("local0", GetArg(create->CmdArgs, "--serial")); UNIT_ASSERT_VALUES_EQUAL( @@ -497,6 +504,10 @@ Y_UNIT_TEST_SUITE(TExternalEndpointTest) UNIT_ASSERT_VALUES_EQUAL("4096", GetArg(create->CmdArgs, "--block-size")); + UNIT_ASSERT_VALUES_EQUAL( + "1111:2222:3333:4444", + GetArg(create->CmdArgs, "--perf-profile")); + UNIT_ASSERT(FindPtr(create->CmdArgs, "--read-only")); auto devices = GetArgN(create->CmdArgs, "--device"); diff --git a/cloud/blockstore/vhost-server/backend.h b/cloud/blockstore/vhost-server/backend.h index 2b339738d4a..ba8413b9291 100644 --- a/cloud/blockstore/vhost-server/backend.h +++ b/cloud/blockstore/vhost-server/backend.h @@ -15,6 +15,7 @@ struct IBackend: public IStartable virtual ~IBackend() = default; virtual vhd_bdev_info Init(const TOptions& options) = 0; + virtual void PrepareStop() = 0; virtual void ProcessQueue( ui32 queueIndex, vhd_request_queue* queue, diff --git a/cloud/blockstore/vhost-server/backend_aio.cpp b/cloud/blockstore/vhost-server/backend_aio.cpp index 07ff6f0317f..5692329a8ed 100644 --- a/cloud/blockstore/vhost-server/backend_aio.cpp +++ b/cloud/blockstore/vhost-server/backend_aio.cpp @@ -157,6 +157,7 @@ class TAioBackend final: public IBackend vhd_bdev_info Init(const TOptions& options) override; void Start() override; + void PrepareStop() override; void Stop() override; void ProcessQueue( ui32 queueIndex, @@ -278,6 +279,10 @@ void TAioBackend::Start() CompletionThread = std::thread([this] { CompletionThreadFunc(); }); } +void TAioBackend::PrepareStop() +{ +} + void TAioBackend::Stop() { STORAGE_INFO("Stopping AIO backend"); diff --git a/cloud/blockstore/vhost-server/backend_null.cpp b/cloud/blockstore/vhost-server/backend_null.cpp index 2ba21e17907..d69a39715a2 100644 --- a/cloud/blockstore/vhost-server/backend_null.cpp +++ b/cloud/blockstore/vhost-server/backend_null.cpp @@ -22,6 +22,7 @@ class TNullBackend final: public IBackend vhd_bdev_info Init(const TOptions& options) override; void Start() override; + void PrepareStop() override; void Stop() override; void ProcessQueue( ui32 queueIndex, @@ -60,6 +61,10 @@ vhd_bdev_info TNullBackend::Init(const TOptions& options) void TNullBackend::Start() {} +void TNullBackend::PrepareStop() +{ +} + void TNullBackend::Stop() {} diff --git a/cloud/blockstore/vhost-server/backend_rdma.cpp b/cloud/blockstore/vhost-server/backend_rdma.cpp index 8e95cc87a86..ecee77c1ec4 100644 --- a/cloud/blockstore/vhost-server/backend_rdma.cpp +++ b/cloud/blockstore/vhost-server/backend_rdma.cpp @@ -1,6 +1,7 @@ #include "backend_rdma.h" #include "backend.h" +#include "throttler.h" #include #include @@ -36,6 +37,7 @@ namespace { //////////////////////////////////////////////////////////////////////////////// constexpr ui32 REQUEST_TIMEOUT_MSEC = 86400000; +constexpr TDuration THROTTLE_DELAY = TDuration::MicroSeconds(100); //////////////////////////////////////////////////////////////////////////////// @@ -132,12 +134,14 @@ class TRdmaBackend final: public IBackend bool ReadOnly = false; ui32 BlockSize = 0; ui32 SectorsToBlockShift = 0; + TVector Throttlers; public: explicit TRdmaBackend(ILoggingServicePtr logging); vhd_bdev_info Init(const TOptions& options) override; void Start() override; + void PrepareStop() override; void Stop() override; void ProcessQueue( ui32 queueIndex, @@ -153,6 +157,9 @@ class TRdmaBackend final: public IBackend TCpuCycles startCycles, bool isError); IBlockStorePtr CreateDataClient(IStoragePtr storage); + void ProcessThrottledRequests(ui32 queueIndex, TSimpleStats& queueStats); + + void ProcessIo(struct vhd_io *io, TSimpleStats& queueStats); }; //////////////////////////////////////////////////////////////////////////////// @@ -171,6 +178,15 @@ vhd_bdev_info TRdmaBackend::Init(const TOptions& options) Scheduler = CreateScheduler(); Timer = CreateWallClockTimer(); + for (ui32 i = 0; i < options.QueueCount; i++) { + Throttlers.push_back(CreateThrottler( + Timer, + options.MaxReadBandwidth / options.QueueCount, + options.MaxReadIops / options.QueueCount, + options.MaxWriteBandwidth / options.QueueCount, + options.MaxWriteIops / options.QueueCount)); + } + ClientId = options.ClientId; ReadOnly = options.ReadOnly; @@ -293,6 +309,13 @@ void TRdmaBackend::Start() DataClient = CreateDataClient(std::move(storage)); } +void TRdmaBackend::PrepareStop() +{ + for (auto& throttler : Throttlers) { + throttler->Stop(); + } +} + void TRdmaBackend::Stop() { STORAGE_INFO("Stopping RDMA backend"); @@ -306,31 +329,60 @@ void TRdmaBackend::ProcessQueue( vhd_request_queue* queue, TSimpleStats& queueStats) { - Y_UNUSED(queueIndex); + auto& throttler = Throttlers[queueIndex]; vhd_request req; while (vhd_dequeue_request(queue, &req)) { + ++queueStats.Dequeued; - struct vhd_bdev_io* bio = vhd_get_bdev_io(req.io); - const TCpuCycles now = GetCycleCount(); - switch (bio->type) { - case VHD_BDEV_READ: - ProcessReadRequest(req.io, now); - ++queueStats.Submitted; - break; - case VHD_BDEV_WRITE: - ProcessWriteRequest(req.io, now); - ++queueStats.Submitted; - break; - default: - STORAGE_ERROR( - "Unexpected vhost request type: " - << static_cast(bio->type)); - vhd_complete_bio(req.io, VHD_BDEV_IOERR); - ++queueStats.SubFailed; - break; + if (throttler->ThrottleIo(req.io)) { + continue; } + + ProcessIo(req.io, queueStats); + } + + ProcessThrottledRequests(queueIndex, queueStats); +} + +void TRdmaBackend::ProcessThrottledRequests( + ui32 queueIndex, + TSimpleStats& queueStats) +{ + auto& throttler = Throttlers[queueIndex]; + + while (throttler->HasThrottledIos()) { + auto* io = throttler->ResumeNextThrottledIo(); + if (!io) { + ::Sleep(THROTTLE_DELAY); + continue; + } + + ProcessIo(io, queueStats); + } +} + +void TRdmaBackend::ProcessIo(struct vhd_io *io, TSimpleStats& queueStats) +{ + struct vhd_bdev_io* bio = vhd_get_bdev_io(io); + const TCpuCycles now = GetCycleCount(); + switch (bio->type) { + case VHD_BDEV_READ: + ProcessReadRequest(io, now); + ++queueStats.Submitted; + break; + case VHD_BDEV_WRITE: + ProcessWriteRequest(io, now); + ++queueStats.Submitted; + break; + default: + STORAGE_ERROR( + "Unexpected vhost request type: " + << static_cast(bio->type)); + vhd_complete_bio(io, VHD_BDEV_IOERR); + ++queueStats.SubFailed; + break; } } diff --git a/cloud/blockstore/vhost-server/options.cpp b/cloud/blockstore/vhost-server/options.cpp index 8fd576a5eb9..275b4d9b213 100644 --- a/cloud/blockstore/vhost-server/options.cpp +++ b/cloud/blockstore/vhost-server/options.cpp @@ -120,6 +120,24 @@ void TOptions::Parse(int argc, char** argv) .RequiredArgument("INT") .StoreResultDef(&RdmaClient.MaxBufferSize); + opts.AddLongOption( + "perf-profile", + "Performance profile " + "(:::)") + .RequiredArgument("STR") + .Handler1T( + [this](TStringBuf s) + { + TVector vals; + Split(s.data(), ":", vals); + Y_ENSURE(vals.size() == 4, "invalid format"); + + MaxReadBandwidth = FromString(vals[0]); + MaxReadIops = FromString(vals[1]); + MaxWriteBandwidth = FromString(vals[2]); + MaxWriteIops = FromString(vals[3]); + }); + TOptsParseResultException res(&opts, argc, argv); if (res.FindLongOptParseResult("verbose") && VerboseLevel.empty()) { diff --git a/cloud/blockstore/vhost-server/options.h b/cloud/blockstore/vhost-server/options.h index b711f5ab57a..abe565e5512 100644 --- a/cloud/blockstore/vhost-server/options.h +++ b/cloud/blockstore/vhost-server/options.h @@ -36,6 +36,11 @@ struct TOptions TString ClientId = "vhost-server"; + ui32 MaxReadBandwidth = 0; + ui32 MaxReadIops = 0; + ui32 MaxWriteBandwidth = 0; + ui32 MaxWriteIops = 0; + struct { ui32 QueueSize = 256; diff --git a/cloud/blockstore/vhost-server/options_ut.cpp b/cloud/blockstore/vhost-server/options_ut.cpp index 7db262e64b2..13e2237f8f1 100644 --- a/cloud/blockstore/vhost-server/options_ut.cpp +++ b/cloud/blockstore/vhost-server/options_ut.cpp @@ -51,6 +51,56 @@ Y_UNIT_TEST_SUITE(TOptionsTest) UNIT_ASSERT_VALUES_EQUAL(1111111, options.Layout[1].Offset); UNIT_ASSERT_VALUES_EQUAL(0, options.Layout[2].Offset); } + + Y_UNIT_TEST(ShouldParseRdmaBackendOptions) + { + TOptions options; + + TVector params { + "binary-path", + "--serial", "id", + "--socket-path", "vhost.sock", + "-q" , "8", + "--disk-id", "disk-id", + "--device-backend", "rdma", + "--block-size", "4096", + "--device", "rdma://host1:port1/uuid1:1111111:0", + "--device", "rdma://host2:port2/uuid2:2222222:0", + "--read-only", + "--perf-profile", "1:2:3:4" + }; + + TVector argv; + for (auto& p: params) { + argv.push_back(&p[0]); + } + + options.Parse(argv.size(), argv.data()); + + UNIT_ASSERT_VALUES_EQUAL("vhost.sock", options.SocketPath); + UNIT_ASSERT_VALUES_EQUAL("id", options.Serial); + UNIT_ASSERT_VALUES_EQUAL("disk-id", options.DiskId); + UNIT_ASSERT(options.ReadOnly); + UNIT_ASSERT(!options.NoSync); + UNIT_ASSERT(!options.NoChmod); + UNIT_ASSERT_VALUES_EQUAL(1024, options.BatchSize); + UNIT_ASSERT_VALUES_EQUAL(8, options.QueueCount); + UNIT_ASSERT_VALUES_EQUAL(2, options.Layout.size()); + UNIT_ASSERT_VALUES_EQUAL("rdma://host1:port1/uuid1", options.Layout[0].DevicePath); + UNIT_ASSERT_VALUES_EQUAL("rdma://host2:port2/uuid2", options.Layout[1].DevicePath); + + UNIT_ASSERT_VALUES_EQUAL(1111111, options.Layout[0].ByteCount); + UNIT_ASSERT_VALUES_EQUAL(2222222, options.Layout[1].ByteCount); + + UNIT_ASSERT_VALUES_EQUAL(0, options.Layout[0].Offset); + UNIT_ASSERT_VALUES_EQUAL(0, options.Layout[1].Offset); + + UNIT_ASSERT_VALUES_EQUAL(1, options.MaxReadBandwidth); + UNIT_ASSERT_VALUES_EQUAL(2, options.MaxReadIops); + UNIT_ASSERT_VALUES_EQUAL(3, options.MaxWriteBandwidth); + UNIT_ASSERT_VALUES_EQUAL(4, options.MaxWriteIops); + } + } } // namespace NCloud::NBlockStore::NVHostServer diff --git a/cloud/blockstore/vhost-server/public.h b/cloud/blockstore/vhost-server/public.h index e71bbdffbbb..8f2fc668f49 100644 --- a/cloud/blockstore/vhost-server/public.h +++ b/cloud/blockstore/vhost-server/public.h @@ -12,4 +12,7 @@ using IBackendPtr = std::shared_ptr; struct ICompletionStats; using ICompletionStatsPtr = std::shared_ptr; +struct IThrottler; +using IThrottlerPtr = std::shared_ptr; + } // namespace NCloud::NBlockStore::NVHostServer diff --git a/cloud/blockstore/vhost-server/server.cpp b/cloud/blockstore/vhost-server/server.cpp index 100eec40dc6..954b43831a4 100644 --- a/cloud/blockstore/vhost-server/server.cpp +++ b/cloud/blockstore/vhost-server/server.cpp @@ -180,6 +180,8 @@ void TServer::Stop() { STORAGE_INFO("Stopping the server"); + Backend->PrepareStop(); + { auto promise = NewPromise(); vhd_unregister_blockdev(Handler, [] (void* opaque) { diff --git a/cloud/blockstore/vhost-server/throttler.cpp b/cloud/blockstore/vhost-server/throttler.cpp new file mode 100644 index 00000000000..00200f4fe2c --- /dev/null +++ b/cloud/blockstore/vhost-server/throttler.cpp @@ -0,0 +1,138 @@ +#include "throttler.h" + +#include + +#include + +namespace NCloud::NBlockStore::NVHostServer { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TThrottler final: public IThrottler +{ +private: + ITimerPtr Timer; + TLeakyBucket Bucket; + ui32 MaxReadBandwidth; + ui32 MaxReadIops; + ui32 MaxWriteBandwidth; + ui32 MaxWriteIops; + TDeque PostponedIos; + bool IsStopped = false; + +public: + TThrottler( + ITimerPtr timer, + ui32 maxReadBandwidth, + ui32 maxReadIops, + ui32 maxWriteBandwidth, + ui32 maxWriteIops); + ~TThrottler() override; + bool HasThrottledIos() const override; + vhd_io* ResumeNextThrottledIo() override; + bool ThrottleIo(struct vhd_io* io) override; + void Stop() override; + +private: + bool IsIoThrottled(struct vhd_io* io); +}; + +//////////////////////////////////////////////////////////////////////////////// + +TThrottler::TThrottler( + ITimerPtr timer, + ui32 maxReadBandwidth, + ui32 maxReadIops, + ui32 maxWriteBandwidth, + ui32 maxWriteIops) + : Timer(std::move(timer)) + , Bucket(1.0, 1.0, 1.0) + , MaxReadBandwidth(maxReadBandwidth) + , MaxReadIops(maxReadIops) + , MaxWriteBandwidth(maxWriteBandwidth) + , MaxWriteIops(maxWriteIops) + +{} + +TThrottler::~TThrottler() +{} + +bool TThrottler::IsIoThrottled(struct vhd_io* io) +{ + if (IsStopped) { + return false; + } + + auto* bio = vhd_get_bdev_io(io); + ui32 maxBandwidth = + bio->type == VHD_BDEV_WRITE ? MaxWriteBandwidth : MaxReadBandwidth; + ui32 maxIops = bio->type == VHD_BDEV_WRITE ? MaxWriteIops : MaxReadIops; + ui64 bytesCount = bio->total_sectors * VHD_SECTOR_SIZE; + + auto seconds = Bucket.Register( + Timer->Now(), + CostPerIO( + CalculateThrottlerC1(maxIops, maxBandwidth), + CalculateThrottlerC2(maxIops, maxBandwidth), + bytesCount).MicroSeconds() / 1e6); + if (seconds == 0) { + return false; + } + + return true; +} + +bool TThrottler::HasThrottledIos() const +{ + return !PostponedIos.empty(); +} + +vhd_io* TThrottler::ResumeNextThrottledIo() +{ + auto io = PostponedIos.front(); + if (!IsIoThrottled(io)) { + PostponedIos.pop_front(); + return io; + } + + return nullptr; +} + +bool TThrottler::ThrottleIo(struct vhd_io* io) +{ + if (IsIoThrottled(io)) { + PostponedIos.push_back(io); + return true; + } + + return false; +} + +void TThrottler::Stop() +{ + IsStopped = true; +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +std::shared_ptr CreateThrottler( + ITimerPtr timer, + ui32 maxReadBandwidth, + ui32 maxReadIops, + ui32 maxWriteBandwidth, + ui32 maxWriteIops) + +{ + return std::make_shared( + std::move(timer), + maxReadBandwidth, + maxReadIops, + maxWriteBandwidth, + maxWriteIops); +} + +} // namespace NCloud::NBlockStore::NVHostServer diff --git a/cloud/blockstore/vhost-server/throttler.h b/cloud/blockstore/vhost-server/throttler.h new file mode 100644 index 00000000000..786baa6fe66 --- /dev/null +++ b/cloud/blockstore/vhost-server/throttler.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +namespace NCloud::NBlockStore::NVHostServer { + +//////////////////////////////////////////////////////////////////////////////// + +struct IThrottler +{ + virtual ~IThrottler() = default; + + virtual bool HasThrottledIos() const = 0; + virtual vhd_io* ResumeNextThrottledIo() = 0; + virtual bool ThrottleIo(struct vhd_io* io) = 0; + virtual void Stop() = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +std::shared_ptr CreateThrottler( + ITimerPtr timer, + ui32 maxReadBandwidth, + ui32 maxReadIops, + ui32 maxWriteBandwidth, + ui32 maxWriteIops); + +} // namespace NCloud::NBlockStore::NVHostServer diff --git a/cloud/blockstore/vhost-server/throttler_ut.cpp b/cloud/blockstore/vhost-server/throttler_ut.cpp new file mode 100644 index 00000000000..df451d9c1f8 --- /dev/null +++ b/cloud/blockstore/vhost-server/throttler_ut.cpp @@ -0,0 +1,145 @@ +#include "throttler.h" + +#include +#include + +#include + +#include + +namespace NCloud::NBlockStore::NVHostServer { + +enum class EIoType +{ + Read, + Write +}; + +/* libvhost virtio-blk private IO structure */ +struct virtio_blk_io +{ + void* opaque[2]; + struct vhd_io io; + struct vhd_bdev_io bdev_io; +}; + +struct virtio_blk_io GetIo(EIoType ioType, size_t size) +{ + virtio_blk_io vbio; + vbio.bdev_io.type = + ioType == EIoType::Read ? VHD_BDEV_READ : VHD_BDEV_WRITE; + vbio.bdev_io.total_sectors = size >> 9; + + return vbio; +} + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TThrottlerTest) +{ + Y_UNIT_TEST(ShouldThrottleRequestsBWCorrectly) + { + auto timer = std::make_shared(); + timer->AdvanceTime(TDuration::Seconds(1)); + + auto checkBW = [&timer](auto throttler, auto vbio) + { + UNIT_ASSERT_VALUES_EQUAL(throttler->ThrottleIo(&vbio.io), false); + UNIT_ASSERT_VALUES_EQUAL(throttler->ThrottleIo(&vbio.io), true); + UNIT_ASSERT_VALUES_EQUAL(throttler->HasThrottledIos(), true); + + timer->AdvanceTime(TDuration::Seconds(1)); + auto resumedIo = throttler->ResumeNextThrottledIo(); + UNIT_ASSERT_VALUES_EQUAL(throttler->HasThrottledIos(), false); + UNIT_ASSERT_VALUES_EQUAL(resumedIo, &vbio.io); + }; + + checkBW( + CreateThrottler( + timer, + 50_MB, // maxReadBandwidth + 50, // maxReadIops + 0, // maxWriteBandwidth + 0 // maxWriteIops + ), + GetIo(EIoType::Read, 40_MB)); + + checkBW( + CreateThrottler( + timer, + 0, // maxReadBandwidth + 0, // maxReadIops + 50_MB, // maxWriteBandwidth + 50 // maxWriteIops + ), + GetIo(EIoType::Write, 40_MB)); + } + + Y_UNIT_TEST(ShouldThrottleRequestsIopsCorrectly) + { + auto timer = std::make_shared(); + timer->AdvanceTime(TDuration::Seconds(1)); + + auto checkIops = [&timer](auto throttler, auto vbio) + { + UNIT_ASSERT_VALUES_EQUAL(throttler->ThrottleIo(&vbio.io), false); + UNIT_ASSERT_VALUES_EQUAL(throttler->ThrottleIo(&vbio.io), false); + UNIT_ASSERT_VALUES_EQUAL(throttler->ThrottleIo(&vbio.io), true); + UNIT_ASSERT_VALUES_EQUAL(throttler->HasThrottledIos(), true); + + timer->AdvanceTime(TDuration::Seconds(1)); + auto resumedIo = throttler->ResumeNextThrottledIo(); + UNIT_ASSERT_VALUES_EQUAL(throttler->HasThrottledIos(), false); + UNIT_ASSERT_VALUES_EQUAL(resumedIo, &vbio.io); + }; + + checkIops( + CreateThrottler( + timer, + 0, // maxReadBandwidth + 2, // maxReadIops + 0, // maxWriteBandwidth + 0 // maxWriteIops + ), + GetIo(EIoType::Read, 40_MB)); + + checkIops( + CreateThrottler( + timer, + 0, // maxReadBandwidth + 0, // maxReadIops + 0, // maxWriteBandwidth + 2 // maxWriteIops + ), + GetIo(EIoType::Write, 40_MB)); + } + + Y_UNIT_TEST(DontThrottleRequestsWhenStopped) + { + auto timer = std::make_shared(); + timer->AdvanceTime(TDuration::Seconds(1)); + + auto vbio = GetIo(EIoType::Write, 40_MB); + auto throttler = CreateThrottler( + timer, + 0, // maxReadBandwidth + 1, // maxReadIops + 0, // maxWriteBandwidth + 1 // maxWriteIops + ); + + UNIT_ASSERT_VALUES_EQUAL(throttler->ThrottleIo(&vbio.io), false); + + for (int i = 0; i < 5; i++) { + UNIT_ASSERT_VALUES_EQUAL(throttler->ThrottleIo(&vbio.io), true); + } + + throttler->Stop(); + + for (int i = 0; i < 5; i++) { + UNIT_ASSERT_VALUES_EQUAL(throttler->ThrottleIo(&vbio.io), false); + } + } +} + +} // namespace NCloud::NBlockStore::NVHostServer diff --git a/cloud/blockstore/vhost-server/ut/ya.make b/cloud/blockstore/vhost-server/ut/ya.make index 4e5ddcd6586..b52bfebf94c 100644 --- a/cloud/blockstore/vhost-server/ut/ya.make +++ b/cloud/blockstore/vhost-server/ut/ya.make @@ -20,6 +20,9 @@ SRCS( stats.cpp stats_ut.cpp + + throttler.cpp + throttler_ut.cpp ) ADDINCL( @@ -30,6 +33,7 @@ PEERDIR( cloud/contrib/vhost cloud/storage/core/libs/common cloud/storage/core/libs/diagnostics + cloud/storage/core/libs/throttling cloud/storage/core/libs/vhost-client library/cpp/getopt diff --git a/cloud/blockstore/vhost-server/ya.make b/cloud/blockstore/vhost-server/ya.make index d068fed7621..21d4f4cdec2 100644 --- a/cloud/blockstore/vhost-server/ya.make +++ b/cloud/blockstore/vhost-server/ya.make @@ -12,6 +12,7 @@ SRCS( request_aio.cpp server.cpp stats.cpp + throttler.cpp ) SPLIT_DWARF()