blockstore loadtests: extracted common TStorageServiceConfig initiali…

…zation code to a function, limiting bw by 32MiB/s in this function, using it in network-ssd/network-hdd loadtests (#828) * blockstore loadtests: extracted common TStorageServiceConfig initialization code to a function, limiting bw by 32MiB/s in this function, using it in network-ssd/network-hdd loadtests * blockstore loadtests: deleted unused imports
ydb-platform · Mar 27, 2024 · 9acbbb0 · 9acbbb0
1 parent fa3c71d
commit 9acbbb0
Show file tree

Hide file tree

Showing 10 changed files with 51 additions and 243 deletions.
diff --git a/cloud/blockstore/tests/loadtest/local-checkpoint/test.py b/cloud/blockstore/tests/loadtest/local-checkpoint/test.py
@@ -3,44 +3,13 @@
 import yatest.common as common
 
 from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig
-from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig
+from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits
 from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest
 from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test
 
-import google.protobuf.json_format as protojson
-
-
-def parse_storage_config(param):
-    if param is None:
-        return None
-
-    return protojson.Parse(param, TStorageServiceConfig())
-
 
 def default_storage_config_patch():
-    bw = 1 << 7     # 128 MB/s
-    iops = 1 << 16
-
-    storage = TStorageServiceConfig()
-    storage.ThrottlingEnabled = True
-
-    storage.SSDUnitReadBandwidth = bw
-    storage.SSDUnitWriteBandwidth = bw
-    storage.SSDMaxReadBandwidth = bw
-    storage.SSDMaxWriteBandwidth = bw
-    storage.SSDUnitReadIops = iops
-    storage.SSDUnitWriteIops = iops
-    storage.SSDMaxReadIops = iops
-    storage.SSDMaxWriteIops = iops
-
-    storage.HDDUnitReadBandwidth = bw
-    storage.HDDUnitWriteBandwidth = bw
-    storage.HDDMaxReadBandwidth = bw
-    storage.HDDMaxWriteBandwidth = bw
-    storage.HDDUnitReadIops = iops
-    storage.HDDUnitWriteIops = iops
-    storage.HDDMaxReadIops = iops
-    storage.HDDMaxWriteIops = iops
+    storage = storage_config_with_default_limits()
 
     storage.InactiveClientsTimeout = 10000
 

diff --git a/cloud/blockstore/tests/loadtest/local-edgecase/test.py b/cloud/blockstore/tests/loadtest/local-edgecase/test.py
@@ -3,35 +3,14 @@
 import yatest.common as common
 
 from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig
-from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig, CT_LOAD
+from cloud.blockstore.config.storage_pb2 import CT_LOAD
+from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits
 from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest
 from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test
 
 
 def default_storage_config_patch(tablet_version=1):
-    bw = 1 << 7     # 128 MB/s
-    iops = 1 << 16
-
-    storage = TStorageServiceConfig()
-    storage.ThrottlingEnabled = True
-
-    storage.SSDUnitReadBandwidth = bw
-    storage.SSDUnitWriteBandwidth = bw
-    storage.SSDMaxReadBandwidth = bw
-    storage.SSDMaxWriteBandwidth = bw
-    storage.SSDUnitReadIops = iops
-    storage.SSDUnitWriteIops = iops
-    storage.SSDMaxReadIops = iops
-    storage.SSDMaxWriteIops = iops
-
-    storage.HDDUnitReadBandwidth = bw
-    storage.HDDUnitWriteBandwidth = bw
-    storage.HDDMaxReadBandwidth = bw
-    storage.HDDMaxWriteBandwidth = bw
-    storage.HDDUnitReadIops = iops
-    storage.HDDUnitWriteIops = iops
-    storage.HDDMaxReadIops = iops
-    storage.HDDMaxWriteIops = iops
+    storage = storage_config_with_default_limits()
 
     if tablet_version == 2:
         storage.BlockDigestsEnabled = True
@@ -44,7 +23,7 @@ def default_storage_config_patch(tablet_version=1):
 
 
 def storage_config_with_batching(tablet_version=1):
-    storage = TStorageServiceConfig()
+    storage = storage_config_with_default_limits()
     storage.WriteRequestBatchingEnabled = True
     storage.ThrottlingEnabled = False
 
@@ -57,7 +36,7 @@ def storage_config_with_batching(tablet_version=1):
 
 
 def storage_config_with_new_compaction():
-    storage = TStorageServiceConfig()
+    storage = storage_config_with_default_limits()
     storage.SSDCompactionType = CT_LOAD
     storage.HDDCompactionType = CT_LOAD
     storage.V1GarbageCompactionEnabled = True

diff --git a/cloud/blockstore/tests/loadtest/local-emergency/test.py b/cloud/blockstore/tests/loadtest/local-emergency/test.py
@@ -3,17 +3,16 @@
 
 import yatest.common as common
 
-from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig
 from cloud.blockstore.public.sdk.python.client import CreateClient, Session
+from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits
 from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest
 from cloud.blockstore.tests.python.lib.test_base import run_test
 
 from contrib.ydb.tests.library.harness.kikimr_runner import get_unique_path_for_current_test, ensure_path_exists
 
 
 def default_storage_config(cache_folder):
-    storage = TStorageServiceConfig()
-    storage.ThrottlingEnabled = True
+    storage = storage_config_with_default_limits()
     storage.HDDSystemChannelPoolKind = "rot"
     storage.SSDSystemChannelPoolKind = "rot"
     storage.HybridSystemChannelPoolKind = "rot"

diff --git a/cloud/blockstore/tests/loadtest/local-endpoints/test.py b/cloud/blockstore/tests/loadtest/local-endpoints/test.py
@@ -5,8 +5,8 @@
 from cloud.blockstore.config.client_pb2 import TClientConfig
 from cloud.blockstore.config.server_pb2 import \
     TServerAppConfig, TServerConfig, TKikimrServiceConfig
-from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig
 
+from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits
 from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest
 from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test
 
@@ -30,30 +30,7 @@ def __init__(
 
 
 def default_storage_config():
-    bw = 1 << 5     # 32 MB/s
-    iops = 1 << 12  # 4096 iops
-
-    storage = TStorageServiceConfig()
-    storage.ThrottlingEnabled = True
-
-    storage.SSDUnitReadBandwidth = bw
-    storage.SSDUnitWriteBandwidth = bw
-    storage.SSDMaxReadBandwidth = bw
-    storage.SSDMaxWriteBandwidth = bw
-    storage.SSDUnitReadIops = iops
-    storage.SSDUnitWriteIops = iops
-    storage.SSDMaxReadIops = iops
-    storage.SSDMaxWriteIops = iops
-
-    storage.HDDUnitReadBandwidth = bw
-    storage.HDDUnitWriteBandwidth = bw
-    storage.HDDMaxReadBandwidth = bw
-    storage.HDDMaxWriteBandwidth = bw
-    storage.HDDUnitReadIops = iops
-    storage.HDDUnitWriteIops = iops
-    storage.HDDMaxReadIops = iops
-    storage.HDDMaxWriteIops = iops
-
+    storage = storage_config_with_default_limits()
     storage.InactiveClientsTimeout = 10000
     return storage
 

diff --git a/cloud/blockstore/tests/loadtest/local-nemesis/test.py b/cloud/blockstore/tests/loadtest/local-nemesis/test.py
@@ -5,41 +5,13 @@
 
 from cloud.blockstore.config.client_pb2 import TClientConfig
 from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig
-from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig
+from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits
 from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest
 from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test, \
     get_restart_interval
 from cloud.storage.core.protos.endpoints_pb2 import EEndpointStorageType
 
 
-def default_storage_config_patch():
-    bw = 1 << 7     # 128 MB/s
-    iops = 1 << 16
-
-    storage = TStorageServiceConfig()
-    storage.ThrottlingEnabled = True
-
-    storage.SSDUnitReadBandwidth = bw
-    storage.SSDUnitWriteBandwidth = bw
-    storage.SSDMaxReadBandwidth = bw
-    storage.SSDMaxWriteBandwidth = bw
-    storage.SSDUnitReadIops = iops
-    storage.SSDUnitWriteIops = iops
-    storage.SSDMaxReadIops = iops
-    storage.SSDMaxWriteIops = iops
-
-    storage.HDDUnitReadBandwidth = bw
-    storage.HDDUnitWriteBandwidth = bw
-    storage.HDDMaxReadBandwidth = bw
-    storage.HDDMaxWriteBandwidth = bw
-    storage.HDDUnitReadIops = iops
-    storage.HDDUnitWriteIops = iops
-    storage.HDDMaxReadIops = iops
-    storage.HDDMaxWriteIops = iops
-
-    return storage
-
-
 class TestCase(object):
 
     def __init__(
@@ -98,7 +70,7 @@ def __run_test(test_case):
     env = LocalLoadTest(
         "",
         server_app_config=server,
-        storage_config_patches=[default_storage_config_patch()],
+        storage_config_patches=[storage_config_with_default_limits()],
         use_in_memory_pdisks=True,
         restart_interval=test_case.restart_interval,
     )

diff --git a/cloud/blockstore/tests/loadtest/local-newfeatures/test.py b/cloud/blockstore/tests/loadtest/local-newfeatures/test.py
@@ -5,7 +5,8 @@
 
 from cloud.blockstore.config.client_pb2 import TClientConfig
 from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig
-from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig, CT_LOAD
+from cloud.blockstore.config.storage_pb2 import CT_LOAD
+from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits
 from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest
 from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test, \
     get_restart_interval
@@ -14,36 +15,11 @@
 
 
 def default_storage_config():
-    bw = 1 << 7     # 128 MB/s
-    iops = 1 << 16
-
-    storage = TStorageServiceConfig()
+    storage = storage_config_with_default_limits()
 
     storage.SSDCompactionType = CT_LOAD
     storage.HDDCompactionType = CT_LOAD
     storage.V1GarbageCompactionEnabled = True
-
-    storage.ThrottlingEnabled = True
-    storage.ThrottlingEnabledSSD = True
-
-    storage.SSDUnitReadBandwidth = bw
-    storage.SSDUnitWriteBandwidth = bw
-    storage.SSDMaxReadBandwidth = bw
-    storage.SSDMaxWriteBandwidth = bw
-    storage.SSDUnitReadIops = iops
-    storage.SSDUnitWriteIops = iops
-    storage.SSDMaxReadIops = iops
-    storage.SSDMaxWriteIops = iops
-
-    storage.HDDUnitReadBandwidth = bw
-    storage.HDDUnitWriteBandwidth = bw
-    storage.HDDMaxReadBandwidth = bw
-    storage.HDDMaxWriteBandwidth = bw
-    storage.HDDUnitReadIops = iops
-    storage.HDDUnitWriteIops = iops
-    storage.HDDMaxReadIops = iops
-    storage.HDDMaxWriteIops = iops
-
     storage.DiskPrefixLengthWithBlockChecksumsInBlobs = 1 << 30
     storage.CheckBlockChecksumsInBlobsUponRead = True
 

diff --git a/cloud/blockstore/tests/loadtest/local-overlay/test.py b/cloud/blockstore/tests/loadtest/local-overlay/test.py
@@ -3,44 +3,13 @@
 import yatest.common as common
 
 from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig
-from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig
+from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits
 from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest
 from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test
 
-import google.protobuf.json_format as protojson
-
-
-def parse_storage_config(param):
-    if param is None:
-        return None
-
-    return protojson.Parse(param, TStorageServiceConfig())
-
 
 def default_storage_config_patch(tablet_version):
-    bw = 1 << 7     # 128 MB/s
-    iops = 1 << 16
-
-    storage = TStorageServiceConfig()
-    storage.ThrottlingEnabled = True
-
-    storage.SSDUnitReadBandwidth = bw
-    storage.SSDUnitWriteBandwidth = bw
-    storage.SSDMaxReadBandwidth = bw
-    storage.SSDMaxWriteBandwidth = bw
-    storage.SSDUnitReadIops = iops
-    storage.SSDUnitWriteIops = iops
-    storage.SSDMaxReadIops = iops
-    storage.SSDMaxWriteIops = iops
-
-    storage.HDDUnitReadBandwidth = bw
-    storage.HDDUnitWriteBandwidth = bw
-    storage.HDDMaxReadBandwidth = bw
-    storage.HDDMaxWriteBandwidth = bw
-    storage.HDDUnitReadIops = iops
-    storage.HDDUnitWriteIops = iops
-    storage.HDDMaxReadIops = iops
-    storage.HDDMaxWriteIops = iops
+    storage = storage_config_with_default_limits()
 
     storage.InactiveClientsTimeout = 10000
 

diff --git a/cloud/blockstore/tests/loadtest/local-v2/test.py b/cloud/blockstore/tests/loadtest/local-v2/test.py
@@ -3,44 +3,13 @@
 import yatest.common as common
 
 from cloud.blockstore.config.server_pb2 import TServerAppConfig, TServerConfig, TKikimrServiceConfig
-from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig
+from cloud.blockstore.tests.python.lib.config import storage_config_with_default_limits
 from cloud.blockstore.tests.python.lib.loadtest_env import LocalLoadTest
 from cloud.blockstore.tests.python.lib.test_base import thread_count, run_test
 
-import google.protobuf.json_format as protojson
-
-
-def parse_storage_config(param):
-    if param is None:
-        return None
-
-    return protojson.Parse(param, TStorageServiceConfig())
-
 
 def default_storage_config_patch():
-    bw = 1 << 7     # 128 MB/s
-    iops = 1 << 16
-
-    storage = TStorageServiceConfig()
-    storage.ThrottlingEnabled = True
-
-    storage.SSDUnitReadBandwidth = bw
-    storage.SSDUnitWriteBandwidth = bw
-    storage.SSDMaxReadBandwidth = bw
-    storage.SSDMaxWriteBandwidth = bw
-    storage.SSDUnitReadIops = iops
-    storage.SSDUnitWriteIops = iops
-    storage.SSDMaxReadIops = iops
-    storage.SSDMaxWriteIops = iops
-
-    storage.HDDUnitReadBandwidth = bw
-    storage.HDDUnitWriteBandwidth = bw
-    storage.HDDMaxReadBandwidth = bw
-    storage.HDDMaxWriteBandwidth = bw
-    storage.HDDUnitReadIops = iops
-    storage.HDDUnitWriteIops = iops
-    storage.HDDMaxReadIops = iops
-    storage.HDDMaxWriteIops = iops
+    storage = storage_config_with_default_limits()
 
     storage.InactiveClientsTimeout = 10000