From 23449dd75088ab781914a6c15c02f2c35d3f2b81 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Mon, 20 Jan 2025 01:06:29 +0100 Subject: [PATCH 01/16] work in progress converting GarnetServer to hosted service --- Directory.Packages.props | 1 + libs/host/Garnet.host.csproj | 13 + libs/host/GarnetApplication.cs | 41 + libs/host/GarnetApplicationBuilder.cs | 46 ++ libs/host/GarnetApplicationOptions.cs | 22 + libs/host/GarnetServer.cs | 732 +++++++++--------- .../host/GarnetServiceCollectionExtensions.cs | 14 + main/GarnetServer/Program.cs | 174 ++--- 8 files changed, 599 insertions(+), 444 deletions(-) create mode 100644 libs/host/GarnetApplication.cs create mode 100644 libs/host/GarnetApplicationBuilder.cs create mode 100644 libs/host/GarnetApplicationOptions.cs create mode 100644 libs/host/GarnetServiceCollectionExtensions.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 0719a40ed8..c11a011937 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -9,6 +9,7 @@ + diff --git a/libs/host/Garnet.host.csproj b/libs/host/Garnet.host.csproj index 1c1259fec3..d16e078906 100644 --- a/libs/host/Garnet.host.csproj +++ b/libs/host/Garnet.host.csproj @@ -20,6 +20,7 @@ + @@ -44,4 +45,16 @@ + + + ..\..\..\..\..\.nuget\packages\microsoft.extensions.diagnostics.abstractions\8.0.0\lib\net8.0\Microsoft.Extensions.Diagnostics.Abstractions.dll + + + ..\..\..\..\..\.nuget\packages\microsoft.extensions.hosting\8.0.0\lib\net8.0\Microsoft.Extensions.Hosting.dll + + + ..\..\..\..\..\.nuget\packages\microsoft.extensions.hosting.abstractions\8.0.0\lib\net8.0\Microsoft.Extensions.Hosting.Abstractions.dll + + + diff --git a/libs/host/GarnetApplication.cs b/libs/host/GarnetApplication.cs new file mode 100644 index 0000000000..6ba3593721 --- /dev/null +++ b/libs/host/GarnetApplication.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; + +namespace Garnet.host; + +/// +/// +/// +public class GarnetApplication : IHost +{ + private readonly IHost host; + + public GarnetApplication(IHost host) + { + this.host = host; + } + + public void Run() + { + HostingAbstractionsHostExtensions.Run(this); + } + + public static GarnetApplicationBuilder CreateHostBuilder() + => new(new()); + + public void Dispose() + => host.Dispose(); + + public Task StartAsync(CancellationToken cancellationToken = default) + => host.StartAsync(cancellationToken); + + public Task StopAsync(CancellationToken cancellationToken = default) + => host.StopAsync(cancellationToken); + + public IServiceProvider Services { get; } +} \ No newline at end of file diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs new file mode 100644 index 0000000000..458afb57ae --- /dev/null +++ b/libs/host/GarnetApplicationBuilder.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.Metrics; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Garnet.host; + +public class GarnetApplicationBuilder : IHostApplicationBuilder +{ + private readonly HostApplicationBuilder hostApplicationBuilder; + + internal GarnetApplicationBuilder(GarnetApplicationOptions options) + { + var configuration = new ConfigurationManager(); + + configuration.AddEnvironmentVariables(prefix: "GARNET_"); + + hostApplicationBuilder = new HostApplicationBuilder(new HostApplicationBuilderSettings + { + Args = options.Args, + ApplicationName = options.ApplicationName, + EnvironmentName = options.EnvironmentName, + Configuration = configuration + }); + } + + public GarnetApplication Build() + { + return new GarnetApplication(hostApplicationBuilder.Build()); + } + + public void ConfigureContainer(IServiceProviderFactory factory, Action configure = null) where TContainerBuilder : notnull => throw new NotImplementedException(); + + public IDictionary Properties { get; } + public IConfigurationManager Configuration { get => hostApplicationBuilder.Configuration; } + public IHostEnvironment Environment { get => hostApplicationBuilder.Environment; } + public ILoggingBuilder Logging { get; } + public IMetricsBuilder Metrics { get; } + public IServiceCollection Services { get; } +} \ No newline at end of file diff --git a/libs/host/GarnetApplicationOptions.cs b/libs/host/GarnetApplicationOptions.cs new file mode 100644 index 0000000000..564229011e --- /dev/null +++ b/libs/host/GarnetApplicationOptions.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Garnet.host; + +public class GarnetApplicationOptions +{ + /// + /// The command line arguments. + /// + public string[] Args { get; init; } + + /// + /// The environment name. + /// + public string EnvironmentName { get; init; } + + /// + /// The application name. + /// + public string ApplicationName { get; init; } +} \ No newline at end of file diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 32389d8aa6..8d19e4ce12 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -8,455 +8,477 @@ using System.Reflection; using System.Text; using System.Threading; +using System.Threading.Tasks; using Garnet.cluster; using Garnet.common; using Garnet.networking; using Garnet.server; using Garnet.server.Auth.Settings; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Tsavorite.core; -namespace Garnet +namespace Garnet; + +using MainStoreAllocator = SpanByteAllocator>; +using MainStoreFunctions = StoreFunctions; + +using ObjectStoreAllocator = GenericAllocator>>; +using ObjectStoreFunctions = StoreFunctions>; + +/// +/// Implementation Garnet server +/// +public class GarnetServer : IHostedService, IDisposable { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; + static readonly string version = GetVersion(); + static string GetVersion() + { + var Version = Assembly.GetExecutingAssembly().GetName().Version; + return $"{Version.Major}.{Version.Minor}.{Version.Build}"; + } - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; + internal GarnetProvider Provider; + + private readonly GarnetServerOptions opts; + private IGarnetServer server; + private TsavoriteKV store; + private TsavoriteKV objectStore; + private IDevice aofDevice; + private TsavoriteLog appendOnlyFile; + private SubscribeBroker> subscribeBroker; + private KVSettings kvSettings; + private KVSettings objKvSettings; + private INamedDeviceFactory logFactory; + private MemoryLogger initLogger; + private ILogger logger; + private readonly ILoggerFactory loggerFactory; + private readonly bool cleanupDir; + private bool disposeLoggerFactory; /// - /// Implementation Garnet server + /// Store and associated information used by this Garnet server /// - public class GarnetServer : IDisposable - { - static readonly string version = GetVersion(); - static string GetVersion() - { - var Version = Assembly.GetExecutingAssembly().GetName().Version; - return $"{Version.Major}.{Version.Minor}.{Version.Build}"; - } + protected StoreWrapper storeWrapper; - internal GarnetProvider Provider; - - private readonly GarnetServerOptions opts; - private IGarnetServer server; - private TsavoriteKV store; - private TsavoriteKV objectStore; - private IDevice aofDevice; - private TsavoriteLog appendOnlyFile; - private SubscribeBroker> subscribeBroker; - private KVSettings kvSettings; - private KVSettings objKvSettings; - private INamedDeviceFactory logFactory; - private MemoryLogger initLogger; - private ILogger logger; - private readonly ILoggerFactory loggerFactory; - private readonly bool cleanupDir; - private bool disposeLoggerFactory; - - /// - /// Store and associated information used by this Garnet server - /// - protected StoreWrapper storeWrapper; - - /// - /// Resp protocol version - /// - readonly string redisProtocolVersion = "7.2.5"; - - /// - /// Metrics API - /// - public MetricsApi Metrics; - - /// - /// Command registration API - /// - public RegisterApi Register; - - /// - /// Store API - /// - public StoreApi Store; - - /// - /// Create Garnet Server instance using specified command line arguments; use Start to start the server. - /// - /// Command line arguments - /// Logger factory - /// Clean up directory. - /// Override for custom authentication settings. - public GarnetServer(string[] commandLineArgs, ILoggerFactory loggerFactory = null, bool cleanupDir = false, IAuthenticationSettings authenticationSettingsOverride = null) - { - Trace.Listeners.Add(new ConsoleTraceListener()); - - // Set up an initial memory logger to log messages from configuration parser into memory. - using (var memLogProvider = new MemoryLoggerProvider()) - { - this.initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); - } + /// + /// Resp protocol version + /// + readonly string redisProtocolVersion = "7.2.5"; - if (!ServerSettingsManager.TryParseCommandLineArguments(commandLineArgs, out var serverSettings, out _, out var exitGracefully, this.initLogger)) - { - if (exitGracefully) - Environment.Exit(0); + /// + /// Metrics API + /// + public MetricsApi Metrics; - // Flush logs from memory logger - FlushMemoryLogger(this.initLogger, "ArgParser", loggerFactory); + /// + /// Command registration API + /// + public RegisterApi Register; - throw new GarnetException("Encountered an error when initializing Garnet server. Please see log messages above for more details."); - } + /// + /// Store API + /// + public StoreApi Store; - if (loggerFactory == null) - { - // If the main logger factory is created by GarnetServer, it should be disposed when GarnetServer is disposed - disposeLoggerFactory = true; - } - else - { - this.initLogger.LogWarning( - $"Received an external ILoggerFactory object. The following configuration options are ignored: {nameof(serverSettings.FileLogger)}, {nameof(serverSettings.LogLevel)}, {nameof(serverSettings.DisableConsoleLogger)}."); - } + /// + /// Create Garnet Server instance using specified command line arguments; use Start to start the server. + /// + /// Command line arguments + /// Logger factory + /// Clean up directory. + /// Override for custom authentication settings. + public GarnetServer(string[] commandLineArgs, ILoggerFactory loggerFactory = null, bool cleanupDir = false, IAuthenticationSettings authenticationSettingsOverride = null) + { + Trace.Listeners.Add(new ConsoleTraceListener()); - // If no logger factory is given, set up main logger factory based on parsed configuration values, - // otherwise use given logger factory. - this.loggerFactory = loggerFactory ?? LoggerFactory.Create(builder => - { - if (!serverSettings.DisableConsoleLogger.GetValueOrDefault()) - { - builder.AddSimpleConsole(options => - { - options.SingleLine = true; - options.TimestampFormat = "hh::mm::ss "; - }); - } - - // Optional: Flush log output to file. - if (serverSettings.FileLogger != null) - builder.AddFile(serverSettings.FileLogger); - builder.SetMinimumLevel(serverSettings.LogLevel); - }); - - // Assign values to GarnetServerOptions - this.opts = serverSettings.GetServerOptions(this.loggerFactory.CreateLogger("Options")); - this.opts.AuthSettings = authenticationSettingsOverride ?? this.opts.AuthSettings; - this.cleanupDir = cleanupDir; - this.InitializeServer(); + // Set up an initial memory logger to log messages from configuration parser into memory. + using (var memLogProvider = new MemoryLoggerProvider()) + { + this.initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); } - /// - /// Create Garnet Server instance using GarnetServerOptions instance; use Start to start the server. - /// - /// Server options - /// Logger factory - /// The IGarnetServer to use. If none is provided, will use a GarnetServerTcp. - /// Whether to clean up data folders on dispose - public GarnetServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, IGarnetServer server = null, bool cleanupDir = false) + if (!ServerSettingsManager.TryParseCommandLineArguments(commandLineArgs, out var serverSettings, out _, out var exitGracefully, this.initLogger)) { - this.server = server; - this.opts = opts; - this.loggerFactory = loggerFactory; - this.cleanupDir = cleanupDir; - this.InitializeServer(); + if (exitGracefully) + Environment.Exit(0); + + // Flush logs from memory logger + FlushMemoryLogger(this.initLogger, "ArgParser", loggerFactory); + + throw new GarnetException("Encountered an error when initializing Garnet server. Please see log messages above for more details."); } - private void InitializeServer() + if (loggerFactory == null) + { + // If the main logger factory is created by GarnetServer, it should be disposed when GarnetServer is disposed + disposeLoggerFactory = true; + } + else { - Debug.Assert(opts != null); + this.initLogger.LogWarning( + $"Received an external ILoggerFactory object. The following configuration options are ignored: {nameof(serverSettings.FileLogger)}, {nameof(serverSettings.LogLevel)}, {nameof(serverSettings.DisableConsoleLogger)}."); + } - if (!opts.QuietMode) + // If no logger factory is given, set up main logger factory based on parsed configuration values, + // otherwise use given logger factory. + this.loggerFactory = loggerFactory ?? LoggerFactory.Create(builder => + { + if (!serverSettings.DisableConsoleLogger.GetValueOrDefault()) { - var red = "\u001b[31m"; - var magenta = "\u001b[35m"; - var normal = "\u001b[0m"; - - Console.WriteLine($@"{red} _________ - /_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(opts.EnableCluster ? "cluster" : "standalone")} mode{red} - '. \ / .' {normal}Port: {opts.Port}{red} - '.\ /.' {magenta}https://aka.ms/GetGarnet{red} - '.' -{normal}"); + builder.AddSimpleConsole(options => + { + options.SingleLine = true; + options.TimestampFormat = "hh::mm::ss "; + }); } - var clusterFactory = opts.EnableCluster ? new ClusterFactory() : null; + // Optional: Flush log output to file. + if (serverSettings.FileLogger != null) + builder.AddFile(serverSettings.FileLogger); + builder.SetMinimumLevel(serverSettings.LogLevel); + }); + + // Assign values to GarnetServerOptions + this.opts = serverSettings.GetServerOptions(this.loggerFactory.CreateLogger("Options")); + this.opts.AuthSettings = authenticationSettingsOverride ?? this.opts.AuthSettings; + this.cleanupDir = cleanupDir; + this.InitializeServer(); + } - this.logger = this.loggerFactory?.CreateLogger("GarnetServer"); - logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", version, IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.Port); + /// + /// Create Garnet Server instance using GarnetServerOptions instance; use Start to start the server. + /// + /// Server options + /// Logger factory + /// The IGarnetServer to use. If none is provided, will use a GarnetServerTcp. + /// Whether to clean up data folders on dispose + public GarnetServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, IGarnetServer server = null, bool cleanupDir = false) + { + this.server = server; + this.opts = opts; + this.loggerFactory = loggerFactory; + this.cleanupDir = cleanupDir; + this.InitializeServer(); + } - // Flush initialization logs from memory logger - FlushMemoryLogger(this.initLogger, "ArgParser", this.loggerFactory); + private void InitializeServer() + { + Debug.Assert(opts != null); - var customCommandManager = new CustomCommandManager(); + if (!opts.QuietMode) + { + var red = "\u001b[31m"; + var magenta = "\u001b[35m"; + var normal = "\u001b[0m"; + + Console.WriteLine($@"{red} _________ +/_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(opts.EnableCluster ? "cluster" : "standalone")} mode{red} +'. \ / .' {normal}Port: {opts.Port}{red} + '.\ /.' {magenta}https://aka.ms/GetGarnet{red} + '.' +{normal}"); + } - var setMax = opts.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads); + var clusterFactory = opts.EnableCluster ? new ClusterFactory() : null; - if (opts.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.ThreadPoolMinThreads, opts.ThreadPoolMinThreads)) - throw new Exception($"Unable to call ThreadPool.SetMinThreads with {opts.ThreadPoolMinThreads}"); + this.logger = this.loggerFactory?.CreateLogger("GarnetServer"); + logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", version, IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.Port); - // Retry to set max threads if it wasn't set in the previous step - if (!setMax && !ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads)) - throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.ThreadPoolMaxThreads}"); + // Flush initialization logs from memory logger + FlushMemoryLogger(this.initLogger, "ArgParser", this.loggerFactory); - CreateMainStore(clusterFactory, out var checkpointDir); - CreateObjectStore(clusterFactory, customCommandManager, checkpointDir, out var objectStoreSizeTracker); + var customCommandManager = new CustomCommandManager(); - if (!opts.DisablePubSub) - subscribeBroker = new SubscribeBroker>(new SpanByteKeySerializer(), null, opts.PubSubPageSizeBytes(), opts.SubscriberRefreshFrequencyMs, true); + var setMax = opts.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads); - CreateAOF(); + if (opts.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.ThreadPoolMinThreads, opts.ThreadPoolMinThreads)) + throw new Exception($"Unable to call ThreadPool.SetMinThreads with {opts.ThreadPoolMinThreads}"); - logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled"); + // Retry to set max threads if it wasn't set in the previous step + if (!setMax && !ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads)) + throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.ThreadPoolMaxThreads}"); - if (logger != null) - { - var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); - if (objectStore != null) - configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0) + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); - logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); - } + CreateMainStore(clusterFactory, out var checkpointDir); + CreateObjectStore(clusterFactory, customCommandManager, checkpointDir, out var objectStoreSizeTracker); - // Create Garnet TCP server if none was provided. - this.server ??= new GarnetServerTcp(opts.Address, opts.Port, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, opts.NetworkConnectionLimit, logger); + if (!opts.DisablePubSub) + subscribeBroker = new SubscribeBroker>(new SpanByteKeySerializer(), null, opts.PubSubPageSizeBytes(), opts.SubscriberRefreshFrequencyMs, true); - storeWrapper = new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, - customCommandManager, appendOnlyFile, opts, clusterFactory: clusterFactory, loggerFactory: loggerFactory); + CreateAOF(); - // Create session provider for Garnet - Provider = new GarnetProvider(storeWrapper, subscribeBroker); + logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled"); - // Create user facing API endpoints - Metrics = new MetricsApi(Provider); - Register = new RegisterApi(Provider); - Store = new StoreApi(storeWrapper); + if (logger != null) + { + var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); + if (objectStore != null) + configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0) + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); + logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); + } - server.Register(WireFormat.ASCII, Provider); + // Create Garnet TCP server if none was provided. + this.server ??= new GarnetServerTcp(opts.Address, opts.Port, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, opts.NetworkConnectionLimit, logger); - LoadModules(customCommandManager); - } + storeWrapper = new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, + customCommandManager, appendOnlyFile, opts, clusterFactory: clusterFactory, loggerFactory: loggerFactory); - private void LoadModules(CustomCommandManager customCommandManager) - { - if (opts.LoadModuleCS == null) - return; + // Create session provider for Garnet + Provider = new GarnetProvider(storeWrapper, subscribeBroker); - foreach (var moduleCS in opts.LoadModuleCS) - { - var moduleCSData = moduleCS.Split(' ', StringSplitOptions.RemoveEmptyEntries); - if (moduleCSData.Length < 1) - continue; + // Create user facing API endpoints + Metrics = new MetricsApi(Provider); + Register = new RegisterApi(Provider); + Store = new StoreApi(storeWrapper); - var modulePath = moduleCSData[0]; - var moduleArgs = moduleCSData.Length > 1 ? moduleCSData.Skip(1).ToArray() : []; - if (ModuleUtils.LoadAssemblies([modulePath], null, true, out var loadedAssemblies, out var errorMsg)) - { - ModuleRegistrar.Instance.LoadModule(customCommandManager, loadedAssemblies.ToList()[0], moduleArgs, logger, out errorMsg); - } - else - { - logger?.LogError("Module {0} failed to load with error {1}", modulePath, Encoding.UTF8.GetString(errorMsg)); - } - } - } + server.Register(WireFormat.ASCII, Provider); - private void CreateMainStore(IClusterFactory clusterFactory, out string checkpointDir) - { - kvSettings = opts.GetSettings(loggerFactory, out logFactory); + LoadModules(customCommandManager); + } - checkpointDir = opts.CheckpointDir ?? opts.LogDir; + private void LoadModules(CustomCommandManager customCommandManager) + { + if (opts.LoadModuleCS == null) + return; - // Run checkpoint on its own thread to control p99 - kvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; - kvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; + foreach (var moduleCS in opts.LoadModuleCS) + { + var moduleCSData = moduleCS.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (moduleCSData.Length < 1) + continue; - var checkpointFactory = opts.DeviceFactoryCreator(); - if (opts.EnableCluster) + var modulePath = moduleCSData[0]; + var moduleArgs = moduleCSData.Length > 1 ? moduleCSData.Skip(1).ToArray() : []; + if (ModuleUtils.LoadAssemblies([modulePath], null, true, out var loadedAssemblies, out var errorMsg)) { - kvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager(checkpointFactory, - new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), isMainStore: true, logger); + ModuleRegistrar.Instance.LoadModule(customCommandManager, loadedAssemblies.ToList()[0], moduleArgs, logger, out errorMsg); } else { - kvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(checkpointFactory, - new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), removeOutdated: true); + logger?.LogError("Module {0} failed to load with error {1}", modulePath, Encoding.UTF8.GetString(errorMsg)); } - - store = new(kvSettings - , StoreFunctions.Create() - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); } + } + + private void CreateMainStore(IClusterFactory clusterFactory, out string checkpointDir) + { + kvSettings = opts.GetSettings(loggerFactory, out logFactory); + + checkpointDir = opts.CheckpointDir ?? opts.LogDir; - private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandManager customCommandManager, string CheckpointDir, out CacheSizeTracker objectStoreSizeTracker) + // Run checkpoint on its own thread to control p99 + kvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; + kvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; + + var checkpointFactory = opts.DeviceFactoryCreator(); + if (opts.EnableCluster) { - objectStoreSizeTracker = null; - if (!opts.DisableObjects) - { - objKvSettings = opts.GetObjectStoreSettings(this.loggerFactory?.CreateLogger("TsavoriteKV [obj]"), - out var objHeapMemorySize, out var objReadCacheHeapMemorySize); - - // Run checkpoint on its own thread to control p99 - objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; - objKvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; - - if (opts.EnableCluster) - objKvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager( - opts.DeviceFactoryCreator(), - new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), - isMainStore: false, logger); - else - objKvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(opts.DeviceFactoryCreator(), - new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), - removeOutdated: true); - - objectStore = new(objKvSettings - , StoreFunctions.Create(new ByteArrayKeyComparer(), - () => new ByteArrayBinaryObjectSerializer(), - () => new GarnetObjectSerializer(customCommandManager)) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - - if (objHeapMemorySize > 0 || objReadCacheHeapMemorySize > 0) - objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objHeapMemorySize, objReadCacheHeapMemorySize, - this.loggerFactory); - } + kvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager(checkpointFactory, + new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), isMainStore: true, logger); + } + else + { + kvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(checkpointFactory, + new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), removeOutdated: true); } - private void CreateAOF() + store = new(kvSettings + , StoreFunctions.Create() + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + } + + private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandManager customCommandManager, string CheckpointDir, out CacheSizeTracker objectStoreSizeTracker) + { + objectStoreSizeTracker = null; + if (!opts.DisableObjects) { - if (opts.EnableAOF) - { - if (opts.MainMemoryReplication && opts.CommitFrequencyMs != -1) - throw new Exception("Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); + objKvSettings = opts.GetObjectStoreSettings(this.loggerFactory?.CreateLogger("TsavoriteKV [obj]"), + out var objHeapMemorySize, out var objReadCacheHeapMemorySize); - opts.GetAofSettings(out var aofSettings); - aofDevice = aofSettings.LogDevice; - appendOnlyFile = new TsavoriteLog(aofSettings, logger: this.loggerFactory?.CreateLogger("TsavoriteLog [aof]")); + // Run checkpoint on its own thread to control p99 + objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; + objKvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; - if (opts.CommitFrequencyMs < 0 && opts.WaitForCommit) - throw new Exception("Cannot use CommitWait with manual commits"); - return; - } + if (opts.EnableCluster) + objKvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager( + opts.DeviceFactoryCreator(), + new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), + isMainStore: false, logger); + else + objKvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(opts.DeviceFactoryCreator(), + new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), + removeOutdated: true); + + objectStore = new(objKvSettings + , StoreFunctions.Create(new ByteArrayKeyComparer(), + () => new ByteArrayBinaryObjectSerializer(), + () => new GarnetObjectSerializer(customCommandManager)) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - if (opts.CommitFrequencyMs != 0 || opts.WaitForCommit) - throw new Exception("Cannot use CommitFrequencyMs or CommitWait without EnableAOF"); + if (objHeapMemorySize > 0 || objReadCacheHeapMemorySize > 0) + objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objHeapMemorySize, objReadCacheHeapMemorySize, + this.loggerFactory); } + } - /// - /// Start server instance - /// - public void Start() + private void CreateAOF() + { + if (opts.EnableAOF) { - Provider.Recover(); - server.Start(); - Provider.Start(); - if (!opts.QuietMode) - Console.WriteLine("* Ready to accept connections"); - } + if (opts.MainMemoryReplication && opts.CommitFrequencyMs != -1) + throw new Exception("Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); - /// - /// Dispose store (including log and checkpoint directory) - /// - public void Dispose() - { - Dispose(cleanupDir); + opts.GetAofSettings(out var aofSettings); + aofDevice = aofSettings.LogDevice; + appendOnlyFile = new TsavoriteLog(aofSettings, logger: this.loggerFactory?.CreateLogger("TsavoriteLog [aof]")); + + if (opts.CommitFrequencyMs < 0 && opts.WaitForCommit) + throw new Exception("Cannot use CommitWait with manual commits"); + return; } - /// - /// Dispose, optionally deleting logs and checkpoints - /// - /// Whether to delete logs and checkpoints - public void Dispose(bool deleteDir = true) + if (opts.CommitFrequencyMs != 0 || opts.WaitForCommit) + throw new Exception("Cannot use CommitFrequencyMs or CommitWait without EnableAOF"); + } + + /// + /// Start server instance + /// + public void Start() + { + Provider.Recover(); + server.Start(); + Provider.Start(); + if (!opts.QuietMode) + Console.WriteLine("* Ready to accept connections"); + } + + /// + /// Dispose store (including log and checkpoint directory) + /// + public void Dispose() + { + Dispose(cleanupDir); + } + + /// + /// Dispose, optionally deleting logs and checkpoints + /// + /// Whether to delete logs and checkpoints + public void Dispose(bool deleteDir = true) + { + InternalDispose(); + if (deleteDir) { - InternalDispose(); - if (deleteDir) + logFactory?.Delete(new FileDescriptor { directoryName = "" }); + if (opts.CheckpointDir != opts.LogDir && !string.IsNullOrEmpty(opts.CheckpointDir)) { - logFactory?.Delete(new FileDescriptor { directoryName = "" }); - if (opts.CheckpointDir != opts.LogDir && !string.IsNullOrEmpty(opts.CheckpointDir)) - { - var ckptdir = opts.DeviceFactoryCreator(); - ckptdir.Initialize(opts.CheckpointDir); - ckptdir.Delete(new FileDescriptor { directoryName = "" }); - } + var ckptdir = opts.DeviceFactoryCreator(); + ckptdir.Initialize(opts.CheckpointDir); + ckptdir.Delete(new FileDescriptor { directoryName = "" }); } } + } - private void InternalDispose() + private void InternalDispose() + { + Provider?.Dispose(); + server.Dispose(); + subscribeBroker?.Dispose(); + store.Dispose(); + appendOnlyFile?.Dispose(); + aofDevice?.Dispose(); + kvSettings.LogDevice?.Dispose(); + if (!opts.DisableObjects) { - Provider?.Dispose(); - server.Dispose(); - subscribeBroker?.Dispose(); - store.Dispose(); - appendOnlyFile?.Dispose(); - aofDevice?.Dispose(); - kvSettings.LogDevice?.Dispose(); - if (!opts.DisableObjects) - { - objectStore.Dispose(); - objKvSettings.LogDevice?.Dispose(); - objKvSettings.ObjectLogDevice?.Dispose(); - } - opts.AuthSettings?.Dispose(); - if (disposeLoggerFactory) - loggerFactory?.Dispose(); + objectStore.Dispose(); + objKvSettings.LogDevice?.Dispose(); + objKvSettings.ObjectLogDevice?.Dispose(); } + opts.AuthSettings?.Dispose(); + if (disposeLoggerFactory) + loggerFactory?.Dispose(); + } + + private static void DeleteDirectory(string path) + { + if (path == null) return; - private static void DeleteDirectory(string path) + // Exceptions may happen due to a handle briefly remaining held after Dispose(). + try { - if (path == null) return; + foreach (string directory in Directory.GetDirectories(path)) + { + DeleteDirectory(directory); + } - // Exceptions may happen due to a handle briefly remaining held after Dispose(). + Directory.Delete(path, true); + } + catch (Exception ex) when (ex is IOException || + ex is UnauthorizedAccessException) + { try { - foreach (string directory in Directory.GetDirectories(path)) - { - DeleteDirectory(directory); - } - Directory.Delete(path, true); } - catch (Exception ex) when (ex is IOException || - ex is UnauthorizedAccessException) - { - try - { - Directory.Delete(path, true); - } - catch { } - } + catch { } } + } - /// - /// Flushes MemoryLogger entries into a destination logger. - /// Destination logger is either created from ILoggerFactory parameter or from a default console logger. - /// - /// The memory logger - /// The category name of the destination logger - /// Optional logger factory for creating the destination logger - private static void FlushMemoryLogger(MemoryLogger memoryLogger, string categoryName, ILoggerFactory dstLoggerFactory = null) - { - if (memoryLogger == null) return; + /// + /// Flushes MemoryLogger entries into a destination logger. + /// Destination logger is either created from ILoggerFactory parameter or from a default console logger. + /// + /// The memory logger + /// The category name of the destination logger + /// Optional logger factory for creating the destination logger + private static void FlushMemoryLogger(MemoryLogger memoryLogger, string categoryName, ILoggerFactory dstLoggerFactory = null) + { + if (memoryLogger == null) return; - // If no logger factory supplied, create a default console logger - var disposeDstLoggerFactory = false; - if (dstLoggerFactory == null) + // If no logger factory supplied, create a default console logger + var disposeDstLoggerFactory = false; + if (dstLoggerFactory == null) + { + dstLoggerFactory = LoggerFactory.Create(builder => builder.AddSimpleConsole(options => { - dstLoggerFactory = LoggerFactory.Create(builder => builder.AddSimpleConsole(options => - { - options.SingleLine = true; - options.TimestampFormat = "hh::mm::ss "; - }).SetMinimumLevel(LogLevel.Information)); - disposeDstLoggerFactory = true; - } + options.SingleLine = true; + options.TimestampFormat = "hh::mm::ss "; + }).SetMinimumLevel(LogLevel.Information)); + disposeDstLoggerFactory = true; + } - // Create the destination logger - var dstLogger = dstLoggerFactory.CreateLogger(categoryName); + // Create the destination logger + var dstLogger = dstLoggerFactory.CreateLogger(categoryName); - // Flush all entries from the memory logger into the destination logger - memoryLogger.FlushLogger(dstLogger); + // Flush all entries from the memory logger into the destination logger + memoryLogger.FlushLogger(dstLogger); - // If a default console logger factory was created, it is no longer needed - if (disposeDstLoggerFactory) - { - dstLoggerFactory.Dispose(); - } + // If a default console logger factory was created, it is no longer needed + if (disposeDstLoggerFactory) + { + dstLoggerFactory.Dispose(); } } + + public Task StartAsync(CancellationToken cancellationToken) + { + Provider.Recover(); + server.Start(); + Provider.Start(); + if (!opts.QuietMode) + Console.WriteLine("* Ready to accept connections"); + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + server?.Dispose(); // or server?.Stop() if you have a separate method + Provider?.Dispose(); + subscribeBroker?.Dispose(); + + // Return completed task + return Task.CompletedTask; + } } \ No newline at end of file diff --git a/libs/host/GarnetServiceCollectionExtensions.cs b/libs/host/GarnetServiceCollectionExtensions.cs new file mode 100644 index 0000000000..c13e61e9d9 --- /dev/null +++ b/libs/host/GarnetServiceCollectionExtensions.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Microsoft.Extensions.DependencyInjection; + +namespace Garnet.host; + +public static class GarnetServiceCollectionExtensions +{ + public static void AddGarnet(this IServiceCollection serviceCollection) + { + + } +} \ No newline at end of file diff --git a/main/GarnetServer/Program.cs b/main/GarnetServer/Program.cs index 314a6a39ba..9521eeb57b 100644 --- a/main/GarnetServer/Program.cs +++ b/main/GarnetServer/Program.cs @@ -3,97 +3,93 @@ using System; using System.Threading; +using Garnet; +using Garnet.host; using Garnet.server; -namespace Garnet +var builder = GarnetApplication.CreateHostBuilder(); + +var app = builder.Build(); + +app.Run(); + +try +{ + using var server = new GarnetServer(args); + + // Optional: register custom extensions + RegisterExtensions(server); + + // Start the server + server.Start(); + + Thread.Sleep(Timeout.Infinite); +} +catch (Exception ex) { - /// - /// Garnet server entry point - /// - class Program + Console.WriteLine($"Unable to initialize server due to exception: {ex.Message}"); +} + +/// +/// Register new commands with the server. You can access these commands from clients using +/// commands such as db.Execute in StackExchange.Redis. Example: +/// db.Execute("SETIFPM", key, value, prefix); +/// +static void RegisterExtensions(GarnetServer server) +{ + // Register custom command on raw strings (SETIFPM = "set if prefix match") + // Add RESP command info to registration for command to appear when client runs COMMAND / COMMAND INFO + var setIfPmCmdInfo = new RespCommandsInfo { - static void Main(string[] args) - { - try - { - using var server = new GarnetServer(args); - - // Optional: register custom extensions - RegisterExtensions(server); - - // Start the server - server.Start(); - - Thread.Sleep(Timeout.Infinite); - } - catch (Exception ex) - { - Console.WriteLine($"Unable to initialize server due to exception: {ex.Message}"); - } - } - - /// - /// Register new commands with the server. You can access these commands from clients using - /// commands such as db.Execute in StackExchange.Redis. Example: - /// db.Execute("SETIFPM", key, value, prefix); - /// - static void RegisterExtensions(GarnetServer server) - { - // Register custom command on raw strings (SETIFPM = "set if prefix match") - // Add RESP command info to registration for command to appear when client runs COMMAND / COMMAND INFO - var setIfPmCmdInfo = new RespCommandsInfo - { - Name = "SETIFPM", - Arity = 4, - FirstKey = 1, - LastKey = 1, - Step = 1, - Flags = RespCommandFlags.DenyOom | RespCommandFlags.Write, - AclCategories = RespAclCategories.String | RespAclCategories.Write, - }; - server.Register.NewCommand("SETIFPM", CommandType.ReadModifyWrite, new SetIfPMCustomCommand(), setIfPmCmdInfo); - - // Register custom command on raw strings (SETWPIFPGT = "set with prefix, if prefix greater than") - server.Register.NewCommand("SETWPIFPGT", CommandType.ReadModifyWrite, new SetWPIFPGTCustomCommand()); - - // Register custom command on raw strings (DELIFM = "delete if value matches") - server.Register.NewCommand("DELIFM", CommandType.ReadModifyWrite, new DeleteIfMatchCustomCommand()); - - // Register custom commands on objects - var factory = new MyDictFactory(); - server.Register.NewType(factory); - server.Register.NewCommand("MYDICTSET", CommandType.ReadModifyWrite, factory, new MyDictSet(), new RespCommandsInfo { Arity = 4 }); - server.Register.NewCommand("MYDICTGET", CommandType.Read, factory, new MyDictGet(), new RespCommandsInfo { Arity = 3 }); - - // Register stored procedure to run a transactional command - // Add RESP command info to registration for command to appear when client runs COMMAND / COMMAND INFO - var readWriteTxCmdInfo = new RespCommandsInfo - { - Name = "READWRITETX", - Arity = 4, - FirstKey = 1, - LastKey = 3, - Step = 1, - Flags = RespCommandFlags.DenyOom | RespCommandFlags.Write, - AclCategories = RespAclCategories.Write, - }; - server.Register.NewTransactionProc("READWRITETX", () => new ReadWriteTxn(), readWriteTxCmdInfo); - - // Register stored procedure to run a transactional command - server.Register.NewTransactionProc("MSETPX", () => new MSetPxTxn()); - - // Register stored procedure to run a transactional command - server.Register.NewTransactionProc("MGETIFPM", () => new MGetIfPM()); - - // Register stored procedure to run a non-transactional command - server.Register.NewTransactionProc("GETTWOKEYSNOTXN", () => new GetTwoKeysNoTxn(), new RespCommandsInfo { Arity = 3 }); - - // Register sample transactional procedures - server.Register.NewTransactionProc("SAMPLEUPDATETX", () => new SampleUpdateTxn(), new RespCommandsInfo { Arity = 9 }); - server.Register.NewTransactionProc("SAMPLEDELETETX", () => new SampleDeleteTxn(), new RespCommandsInfo { Arity = 6 }); - - server.Register.NewProcedure("SUM", () => new Sum()); - server.Register.NewProcedure("SETMAINANDOBJECT", () => new SetStringAndList()); - } - } + Name = "SETIFPM", + Arity = 4, + FirstKey = 1, + LastKey = 1, + Step = 1, + Flags = RespCommandFlags.DenyOom | RespCommandFlags.Write, + AclCategories = RespAclCategories.String | RespAclCategories.Write, + }; + server.Register.NewCommand("SETIFPM", CommandType.ReadModifyWrite, new SetIfPMCustomCommand(), setIfPmCmdInfo); + + // Register custom command on raw strings (SETWPIFPGT = "set with prefix, if prefix greater than") + server.Register.NewCommand("SETWPIFPGT", CommandType.ReadModifyWrite, new SetWPIFPGTCustomCommand()); + + // Register custom command on raw strings (DELIFM = "delete if value matches") + server.Register.NewCommand("DELIFM", CommandType.ReadModifyWrite, new DeleteIfMatchCustomCommand()); + + // Register custom commands on objects + var factory = new MyDictFactory(); + server.Register.NewType(factory); + server.Register.NewCommand("MYDICTSET", CommandType.ReadModifyWrite, factory, new MyDictSet(), new RespCommandsInfo { Arity = 4 }); + server.Register.NewCommand("MYDICTGET", CommandType.Read, factory, new MyDictGet(), new RespCommandsInfo { Arity = 3 }); + + // Register stored procedure to run a transactional command + // Add RESP command info to registration for command to appear when client runs COMMAND / COMMAND INFO + var readWriteTxCmdInfo = new RespCommandsInfo + { + Name = "READWRITETX", + Arity = 4, + FirstKey = 1, + LastKey = 3, + Step = 1, + Flags = RespCommandFlags.DenyOom | RespCommandFlags.Write, + AclCategories = RespAclCategories.Write, + }; + server.Register.NewTransactionProc("READWRITETX", () => new ReadWriteTxn(), readWriteTxCmdInfo); + + // Register stored procedure to run a transactional command + server.Register.NewTransactionProc("MSETPX", () => new MSetPxTxn()); + + // Register stored procedure to run a transactional command + server.Register.NewTransactionProc("MGETIFPM", () => new MGetIfPM()); + + // Register stored procedure to run a non-transactional command + server.Register.NewTransactionProc("GETTWOKEYSNOTXN", () => new GetTwoKeysNoTxn(), new RespCommandsInfo { Arity = 3 }); + + // Register sample transactional procedures + server.Register.NewTransactionProc("SAMPLEUPDATETX", () => new SampleUpdateTxn(), new RespCommandsInfo { Arity = 9 }); + server.Register.NewTransactionProc("SAMPLEDELETETX", () => new SampleDeleteTxn(), new RespCommandsInfo { Arity = 6 }); + + server.Register.NewProcedure("SUM", () => new Sum()); + server.Register.NewProcedure("SETMAINANDOBJECT", () => new SetStringAndList()); } \ No newline at end of file From d87366f0cbf2bbfe7a67c1b137365e39d4108beb Mon Sep 17 00:00:00 2001 From: s3w3nofficial Date: Mon, 20 Jan 2025 01:20:59 +0100 Subject: [PATCH 02/16] continue with conversion --- Directory.Packages.props | 1 + libs/host/Garnet.host.csproj | 15 ++------------- libs/host/GarnetApplication.cs | 6 ++++++ libs/host/GarnetApplicationBuilder.cs | 3 +++ main/GarnetServer/Program.cs | 4 +++- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index c11a011937..bc41f0c970 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -10,6 +10,7 @@ + diff --git a/libs/host/Garnet.host.csproj b/libs/host/Garnet.host.csproj index d16e078906..c8d401b6b6 100644 --- a/libs/host/Garnet.host.csproj +++ b/libs/host/Garnet.host.csproj @@ -21,6 +21,8 @@ + + @@ -44,17 +46,4 @@ - - - - ..\..\..\..\..\.nuget\packages\microsoft.extensions.diagnostics.abstractions\8.0.0\lib\net8.0\Microsoft.Extensions.Diagnostics.Abstractions.dll - - - ..\..\..\..\..\.nuget\packages\microsoft.extensions.hosting\8.0.0\lib\net8.0\Microsoft.Extensions.Hosting.dll - - - ..\..\..\..\..\.nuget\packages\microsoft.extensions.hosting.abstractions\8.0.0\lib\net8.0\Microsoft.Extensions.Hosting.Abstractions.dll - - - diff --git a/libs/host/GarnetApplication.cs b/libs/host/GarnetApplication.cs index 6ba3593721..4086c24e7e 100644 --- a/libs/host/GarnetApplication.cs +++ b/libs/host/GarnetApplication.cs @@ -27,6 +27,12 @@ public void Run() public static GarnetApplicationBuilder CreateHostBuilder() => new(new()); + + public static GarnetApplicationBuilder CreateHostBuilder(string[] args) + => new(new() + { + Args = args + }); public void Dispose() => host.Dispose(); diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 458afb57ae..5e6717ddc6 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Garnet.server; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.Metrics; @@ -28,6 +29,8 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) EnvironmentName = options.EnvironmentName, Configuration = configuration }); + + hostApplicationBuilder.Services.AddHostedService(_ => new GarnetServer(options.Args)); } public GarnetApplication Build() diff --git a/main/GarnetServer/Program.cs b/main/GarnetServer/Program.cs index 9521eeb57b..ec2bffd2e1 100644 --- a/main/GarnetServer/Program.cs +++ b/main/GarnetServer/Program.cs @@ -7,12 +7,13 @@ using Garnet.host; using Garnet.server; -var builder = GarnetApplication.CreateHostBuilder(); +var builder = GarnetApplication.CreateHostBuilder(args); var app = builder.Build(); app.Run(); +/* try { using var server = new GarnetServer(args); @@ -29,6 +30,7 @@ { Console.WriteLine($"Unable to initialize server due to exception: {ex.Message}"); } +*/ /// /// Register new commands with the server. You can access these commands from clients using From da4636080023629fe433eb5f8c71888e9a561331 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Mon, 20 Jan 2025 06:51:36 +0100 Subject: [PATCH 03/16] fix stopping --- libs/host/GarnetApplication.cs | 28 ++++++++------- libs/host/GarnetApplicationBuilder.cs | 51 ++++++++++++++++++++++----- libs/host/GarnetServer.cs | 17 ++++----- main/GarnetServer/Program.cs | 21 ----------- 4 files changed, 64 insertions(+), 53 deletions(-) diff --git a/libs/host/GarnetApplication.cs b/libs/host/GarnetApplication.cs index 4086c24e7e..91a19cdc50 100644 --- a/libs/host/GarnetApplication.cs +++ b/libs/host/GarnetApplication.cs @@ -19,12 +19,27 @@ public GarnetApplication(IHost host) { this.host = host; } + + public IServiceProvider Services => host.Services; - public void Run() + public Task StartAsync(CancellationToken cancellationToken = default) + => host.StartAsync(cancellationToken); + + public Task StopAsync(CancellationToken cancellationToken = default) + => host.StopAsync(cancellationToken); + + public void Dispose() => host.Dispose(); + + public void Run(CancellationToken cancellationToken = default) { HostingAbstractionsHostExtensions.Run(this); } + public async Task RunAsync(CancellationToken cancellationToken = default) + { + await HostingAbstractionsHostExtensions.RunAsync(this, cancellationToken); + } + public static GarnetApplicationBuilder CreateHostBuilder() => new(new()); @@ -33,15 +48,4 @@ public static GarnetApplicationBuilder CreateHostBuilder(string[] args) { Args = args }); - - public void Dispose() - => host.Dispose(); - - public Task StartAsync(CancellationToken cancellationToken = default) - => host.StartAsync(cancellationToken); - - public Task StopAsync(CancellationToken cancellationToken = default) - => host.StopAsync(cancellationToken); - - public IServiceProvider Services { get; } } \ No newline at end of file diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 5e6717ddc6..610192e2ab 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -30,20 +30,53 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) Configuration = configuration }); - hostApplicationBuilder.Services.AddHostedService(_ => new GarnetServer(options.Args)); + /* + hostApplicationBuilder.Logging.ClearProviders(); + hostApplicationBuilder.Logging.AddConsole(); + hostApplicationBuilder.Logging.SetMinimumLevel(LogLevel.Trace); + */ + + var garnetOptions = new GarnetServerOptions + { + Port = 6379, + QuietMode = false + }; + + hostApplicationBuilder.Services.AddHostedService(sp => + { + var loggerFactory = sp.GetRequiredService(); + //return new GarnetServer(garnetOptions, loggerFactory, null, cleanupDir: false); + return new GarnetServer(options.Args, loggerFactory, false); + }); } public GarnetApplication Build() { - return new GarnetApplication(hostApplicationBuilder.Build()); + var host = hostApplicationBuilder.Build(); + return new GarnetApplication(host); + } + + public void ConfigureContainer(IServiceProviderFactory factory, + Action configure = null) + where TContainerBuilder : notnull + { + hostApplicationBuilder.ConfigureContainer(factory, configure); } - - public void ConfigureContainer(IServiceProviderFactory factory, Action configure = null) where TContainerBuilder : notnull => throw new NotImplementedException(); public IDictionary Properties { get; } - public IConfigurationManager Configuration { get => hostApplicationBuilder.Configuration; } - public IHostEnvironment Environment { get => hostApplicationBuilder.Environment; } - public ILoggingBuilder Logging { get; } - public IMetricsBuilder Metrics { get; } - public IServiceCollection Services { get; } + + public IConfigurationManager Configuration + => hostApplicationBuilder.Configuration; + + public IHostEnvironment Environment + => hostApplicationBuilder.Environment; + + public ILoggingBuilder Logging + => hostApplicationBuilder.Logging; + + public IMetricsBuilder Metrics + => hostApplicationBuilder.Metrics; + + public IServiceCollection Services + => hostApplicationBuilder.Services; } \ No newline at end of file diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 8d19e4ce12..c3cbc4c66d 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -382,7 +382,9 @@ public void Dispose(bool deleteDir = true) private void InternalDispose() { - Provider?.Dispose(); + // Provider.Dispose will get stuck + //Provider?.Dispose(); + server.Dispose(); subscribeBroker?.Dispose(); store.Dispose(); @@ -463,22 +465,15 @@ private static void FlushMemoryLogger(MemoryLogger memoryLogger, string category public Task StartAsync(CancellationToken cancellationToken) { - Provider.Recover(); - server.Start(); - Provider.Start(); - if (!opts.QuietMode) - Console.WriteLine("* Ready to accept connections"); + Start(); return Task.CompletedTask; } public Task StopAsync(CancellationToken cancellationToken) { - server?.Dispose(); // or server?.Stop() if you have a separate method - Provider?.Dispose(); - subscribeBroker?.Dispose(); - - // Return completed task + Dispose(); + return Task.CompletedTask; } } \ No newline at end of file diff --git a/main/GarnetServer/Program.cs b/main/GarnetServer/Program.cs index ec2bffd2e1..b7e71ce22c 100644 --- a/main/GarnetServer/Program.cs +++ b/main/GarnetServer/Program.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; -using System.Threading; using Garnet; using Garnet.host; using Garnet.server; @@ -13,25 +11,6 @@ app.Run(); -/* -try -{ - using var server = new GarnetServer(args); - - // Optional: register custom extensions - RegisterExtensions(server); - - // Start the server - server.Start(); - - Thread.Sleep(Timeout.Infinite); -} -catch (Exception ex) -{ - Console.WriteLine($"Unable to initialize server due to exception: {ex.Message}"); -} -*/ - /// /// Register new commands with the server. You can access these commands from clients using /// commands such as db.Execute in StackExchange.Redis. Example: From a74f2128498c5f8f9da5c44703aee58536f2685a Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Mon, 20 Jan 2025 07:08:38 +0100 Subject: [PATCH 04/16] fix logging --- libs/host/GarnetApplicationBuilder.cs | 13 +++-- libs/host/GarnetServer.cs | 68 +++++++++++++++------------ 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 610192e2ab..a274274828 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -30,24 +30,29 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) Configuration = configuration }); - /* hostApplicationBuilder.Logging.ClearProviders(); - hostApplicationBuilder.Logging.AddConsole(); - hostApplicationBuilder.Logging.SetMinimumLevel(LogLevel.Trace); - */ + hostApplicationBuilder.Logging.AddSimpleConsole(simpleConsoleFormatterOptions => + { + simpleConsoleFormatterOptions.SingleLine = true; + simpleConsoleFormatterOptions.TimestampFormat = "hh::mm::ss "; + }); var garnetOptions = new GarnetServerOptions { Port = 6379, QuietMode = false }; + + hostApplicationBuilder.Services.AddHostedService(); + /* hostApplicationBuilder.Services.AddHostedService(sp => { var loggerFactory = sp.GetRequiredService(); //return new GarnetServer(garnetOptions, loggerFactory, null, cleanupDir: false); return new GarnetServer(options.Args, loggerFactory, false); }); + */ } public GarnetApplication Build() diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index c3cbc4c66d..43f228e74c 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -81,6 +81,36 @@ static string GetVersion() /// public StoreApi Store; + public GarnetServer(ILogger logger) + { + this.logger = logger; + Trace.Listeners.Add(new ConsoleTraceListener()); + + // Set up an initial memory logger to log messages from configuration parser into memory. + using (var memLogProvider = new MemoryLoggerProvider()) + { + this.initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); + } + + if (!ServerSettingsManager.TryParseCommandLineArguments([], out var serverSettings, out _, out var exitGracefully, this.initLogger)) + { + if (exitGracefully) + Environment.Exit(0); + + // Flush logs from memory logger + FlushMemoryLogger(this.initLogger, "ArgParser", loggerFactory); + + throw new GarnetException("Encountered an error when initializing Garnet server. Please see log messages above for more details."); + } + + // Assign values to GarnetServerOptions + this.opts = serverSettings.GetServerOptions(this.logger); + this.opts.AuthSettings = this.opts.AuthSettings; + this.cleanupDir = cleanupDir; + this.InitializeServer(); + } + + /* /// /// Create Garnet Server instance using specified command line arguments; use Start to start the server. /// @@ -161,6 +191,7 @@ public GarnetServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = nul this.cleanupDir = cleanupDir; this.InitializeServer(); } + */ private void InitializeServer() { @@ -172,12 +203,12 @@ private void InitializeServer() var magenta = "\u001b[35m"; var normal = "\u001b[0m"; - Console.WriteLine($@"{red} _________ -/_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(opts.EnableCluster ? "cluster" : "standalone")} mode{red} -'. \ / .' {normal}Port: {opts.Port}{red} - '.\ /.' {magenta}https://aka.ms/GetGarnet{red} - '.' -{normal}"); + Console.WriteLine($@"{red} _________ + /_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(opts.EnableCluster ? "cluster" : "standalone")} mode{red} + '. \ / .' {normal}Port: {opts.Port}{red} + '.\ /.' {magenta}https://aka.ms/GetGarnet{red} + '.' + {normal}"); } var clusterFactory = opts.EnableCluster ? new ClusterFactory() : null; @@ -402,31 +433,6 @@ private void InternalDispose() loggerFactory?.Dispose(); } - private static void DeleteDirectory(string path) - { - if (path == null) return; - - // Exceptions may happen due to a handle briefly remaining held after Dispose(). - try - { - foreach (string directory in Directory.GetDirectories(path)) - { - DeleteDirectory(directory); - } - - Directory.Delete(path, true); - } - catch (Exception ex) when (ex is IOException || - ex is UnauthorizedAccessException) - { - try - { - Directory.Delete(path, true); - } - catch { } - } - } - /// /// Flushes MemoryLogger entries into a destination logger. /// Destination logger is either created from ILoggerFactory parameter or from a default console logger. From 95bd154fa763c625dfd39411febf6eddfb92f167 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Mon, 20 Jan 2025 08:01:27 +0100 Subject: [PATCH 05/16] register garnet tcp server to di --- libs/cluster/ClusterFactory.cs | 30 +++--- libs/host/GarnetApplicationBuilder.cs | 34 +++++++ libs/host/GarnetServer.cs | 129 +++++++++++-------------- libs/host/MemoryLogger.cs | 4 +- libs/server/Cluster/IClusterFactory.cs | 2 +- 5 files changed, 111 insertions(+), 88 deletions(-) diff --git a/libs/cluster/ClusterFactory.cs b/libs/cluster/ClusterFactory.cs index 8a8ef903e6..f4e85a3c21 100644 --- a/libs/cluster/ClusterFactory.cs +++ b/libs/cluster/ClusterFactory.cs @@ -5,19 +5,25 @@ using Microsoft.Extensions.Logging; using Tsavorite.core; -namespace Garnet.cluster +namespace Garnet.cluster; + +/// +/// Cluster factory +/// +public class ClusterFactory : IClusterFactory { - /// - /// Cluster factory - /// - public class ClusterFactory : IClusterFactory + private readonly ILogger logger; + + public ClusterFactory(ILogger logger) { - /// - public DeviceLogCommitCheckpointManager CreateCheckpointManager(INamedDeviceFactory deviceFactory, ICheckpointNamingScheme checkpointNamingScheme, bool isMainStore, ILogger logger = default) - => new ReplicationLogCheckpointManager(deviceFactory, checkpointNamingScheme, isMainStore, logger: logger); - - /// - public IClusterProvider CreateClusterProvider(StoreWrapper store) - => new ClusterProvider(store); + this.logger = logger; } + + /// + public DeviceLogCommitCheckpointManager CreateCheckpointManager(INamedDeviceFactory deviceFactory, ICheckpointNamingScheme checkpointNamingScheme, bool isMainStore) + => new ReplicationLogCheckpointManager(deviceFactory, checkpointNamingScheme, isMainStore, logger: logger); + + /// + public IClusterProvider CreateClusterProvider(StoreWrapper store) + => new ClusterProvider(store); } \ No newline at end of file diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index a274274828..e6c3efa630 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -3,6 +3,8 @@ using System; using System.Collections.Generic; +using Garnet.cluster; +using Garnet.common; using Garnet.server; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -42,6 +44,38 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) Port = 6379, QuietMode = false }; + + hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddSingleton(sp => + { + MemoryLogger initLogger; + + // Set up an initial memory logger to log messages from configuration parser into memory. + using (var memLogProvider = new MemoryLoggerProvider()) + { + initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); + } + + if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, + out var exitGracefully, initLogger)) + { + if (exitGracefully) + System.Environment.Exit(0); + + // Flush logs from memory logger + //FlushMemoryLogger(this.initLogger, "ArgParser"); + + throw new GarnetException( + "Encountered an error when initializing Garnet server. Please see log messages above for more details."); + } + + var loggerFactory = sp.GetRequiredService(); + var logger = loggerFactory.CreateLogger("Garnet"); + var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); + + return new GarnetServerTcp(opts.Address, opts.Port, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, + opts.NetworkConnectionLimit, logger); + }); hostApplicationBuilder.Services.AddHostedService(); diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 43f228e74c..ebf20d755b 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -3,17 +3,14 @@ using System; using System.Diagnostics; -using System.IO; using System.Linq; using System.Reflection; using System.Text; using System.Threading; using System.Threading.Tasks; -using Garnet.cluster; using Garnet.common; using Garnet.networking; using Garnet.server; -using Garnet.server.Auth.Settings; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -50,11 +47,12 @@ static string GetVersion() private KVSettings kvSettings; private KVSettings objKvSettings; private INamedDeviceFactory logFactory; - private MemoryLogger initLogger; - private ILogger logger; + private readonly MemoryLogger initLogger; + private readonly ILogger logger; private readonly ILoggerFactory loggerFactory; private readonly bool cleanupDir; private bool disposeLoggerFactory; + private readonly IClusterFactory clusterFactory; /// /// Store and associated information used by this Garnet server @@ -81,9 +79,17 @@ static string GetVersion() /// public StoreApi Store; - public GarnetServer(ILogger logger) + public GarnetServer( + ILogger logger, + ILoggerFactory loggerFactory, + IClusterFactory clusterFactory, + GarnetServerTcp garnetServerTcp) { this.logger = logger; + this.loggerFactory = loggerFactory; + this.clusterFactory = clusterFactory; + this.server = garnetServerTcp; + Trace.Listeners.Add(new ConsoleTraceListener()); // Set up an initial memory logger to log messages from configuration parser into memory. @@ -98,13 +104,13 @@ public GarnetServer(ILogger logger) Environment.Exit(0); // Flush logs from memory logger - FlushMemoryLogger(this.initLogger, "ArgParser", loggerFactory); + FlushMemoryLogger(this.initLogger, "ArgParser"); throw new GarnetException("Encountered an error when initializing Garnet server. Please see log messages above for more details."); } // Assign values to GarnetServerOptions - this.opts = serverSettings.GetServerOptions(this.logger); + this.opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); this.opts.AuthSettings = this.opts.AuthSettings; this.cleanupDir = cleanupDir; this.InitializeServer(); @@ -211,60 +217,59 @@ private void InitializeServer() {normal}"); } - var clusterFactory = opts.EnableCluster ? new ClusterFactory() : null; + var clusterFactory = opts.EnableCluster ? this.clusterFactory : null; - this.logger = this.loggerFactory?.CreateLogger("GarnetServer"); - logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", version, IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.Port); + using (logger.BeginScope("GarnetServer")) + { + logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", version, IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.Port); - // Flush initialization logs from memory logger - FlushMemoryLogger(this.initLogger, "ArgParser", this.loggerFactory); + // Flush initialization logs from memory logger + FlushMemoryLogger(this.initLogger, "ArgParser"); - var customCommandManager = new CustomCommandManager(); + var customCommandManager = new CustomCommandManager(); - var setMax = opts.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads); + var setMax = opts.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads); - if (opts.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.ThreadPoolMinThreads, opts.ThreadPoolMinThreads)) - throw new Exception($"Unable to call ThreadPool.SetMinThreads with {opts.ThreadPoolMinThreads}"); + if (opts.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.ThreadPoolMinThreads, opts.ThreadPoolMinThreads)) + throw new Exception($"Unable to call ThreadPool.SetMinThreads with {opts.ThreadPoolMinThreads}"); - // Retry to set max threads if it wasn't set in the previous step - if (!setMax && !ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads)) - throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.ThreadPoolMaxThreads}"); + // Retry to set max threads if it wasn't set in the previous step + if (!setMax && !ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads)) + throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.ThreadPoolMaxThreads}"); - CreateMainStore(clusterFactory, out var checkpointDir); - CreateObjectStore(clusterFactory, customCommandManager, checkpointDir, out var objectStoreSizeTracker); + CreateMainStore(clusterFactory, out var checkpointDir); + CreateObjectStore(clusterFactory, customCommandManager, checkpointDir, out var objectStoreSizeTracker); - if (!opts.DisablePubSub) - subscribeBroker = new SubscribeBroker>(new SpanByteKeySerializer(), null, opts.PubSubPageSizeBytes(), opts.SubscriberRefreshFrequencyMs, true); + if (!opts.DisablePubSub) + subscribeBroker = new SubscribeBroker>(new SpanByteKeySerializer(), null, opts.PubSubPageSizeBytes(), opts.SubscriberRefreshFrequencyMs, true); - CreateAOF(); + CreateAOF(); - logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled"); + logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled"); - if (logger != null) - { - var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); - if (objectStore != null) - configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0) + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); - logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); - } - - // Create Garnet TCP server if none was provided. - this.server ??= new GarnetServerTcp(opts.Address, opts.Port, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, opts.NetworkConnectionLimit, logger); + if (logger != null) + { + var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); + if (objectStore != null) + configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0) + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); + logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); + } - storeWrapper = new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, - customCommandManager, appendOnlyFile, opts, clusterFactory: clusterFactory, loggerFactory: loggerFactory); + storeWrapper = new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, + customCommandManager, appendOnlyFile, opts, clusterFactory: clusterFactory, loggerFactory: loggerFactory); - // Create session provider for Garnet - Provider = new GarnetProvider(storeWrapper, subscribeBroker); + // Create session provider for Garnet + Provider = new GarnetProvider(storeWrapper, subscribeBroker); - // Create user facing API endpoints - Metrics = new MetricsApi(Provider); - Register = new RegisterApi(Provider); - Store = new StoreApi(storeWrapper); + // Create user facing API endpoints + Metrics = new MetricsApi(Provider); + Register = new RegisterApi(Provider); + Store = new StoreApi(storeWrapper); - server.Register(WireFormat.ASCII, Provider); + server.Register(WireFormat.ASCII, Provider); - LoadModules(customCommandManager); + LoadModules(customCommandManager); + } } private void LoadModules(CustomCommandManager customCommandManager) @@ -305,7 +310,7 @@ private void CreateMainStore(IClusterFactory clusterFactory, out string checkpoi if (opts.EnableCluster) { kvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager(checkpointFactory, - new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), isMainStore: true, logger); + new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), isMainStore: true); } else { @@ -334,7 +339,7 @@ private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandMana objKvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager( opts.DeviceFactoryCreator(), new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), - isMainStore: false, logger); + isMainStore: false); else objKvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(opts.DeviceFactoryCreator(), new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), @@ -439,33 +444,11 @@ private void InternalDispose() /// /// The memory logger /// The category name of the destination logger - /// Optional logger factory for creating the destination logger - private static void FlushMemoryLogger(MemoryLogger memoryLogger, string categoryName, ILoggerFactory dstLoggerFactory = null) + private void FlushMemoryLogger(MemoryLogger memoryLogger, string categoryName) { - if (memoryLogger == null) return; - - // If no logger factory supplied, create a default console logger - var disposeDstLoggerFactory = false; - if (dstLoggerFactory == null) - { - dstLoggerFactory = LoggerFactory.Create(builder => builder.AddSimpleConsole(options => - { - options.SingleLine = true; - options.TimestampFormat = "hh::mm::ss "; - }).SetMinimumLevel(LogLevel.Information)); - disposeDstLoggerFactory = true; - } - - // Create the destination logger - var dstLogger = dstLoggerFactory.CreateLogger(categoryName); - - // Flush all entries from the memory logger into the destination logger - memoryLogger.FlushLogger(dstLogger); - - // If a default console logger factory was created, it is no longer needed - if (disposeDstLoggerFactory) + using (this.logger.BeginScope(categoryName)) { - dstLoggerFactory.Dispose(); + memoryLogger.FlushLogger(this.logger); } } diff --git a/libs/host/MemoryLogger.cs b/libs/host/MemoryLogger.cs index f5e4efbf73..720fabfe6b 100644 --- a/libs/host/MemoryLogger.cs +++ b/libs/host/MemoryLogger.cs @@ -34,12 +34,12 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except /// Flushes logger entries into a destination logger. /// /// The logger to which to flush log entries - public void FlushLogger(ILogger dstLogger) + public void FlushLogger(ILogger logger) { foreach (var entry in this._memoryLog) { #pragma warning disable CA2254 // Template should be a static expression - dstLogger.Log(entry.Item1, entry.Item2, entry.Item3); + logger.Log(entry.Item1, entry.Item2, entry.Item3); #pragma warning restore CA2254 // Template should be a static expression } this._memoryLog.Clear(); diff --git a/libs/server/Cluster/IClusterFactory.cs b/libs/server/Cluster/IClusterFactory.cs index 675afe876d..82a62c2d34 100644 --- a/libs/server/Cluster/IClusterFactory.cs +++ b/libs/server/Cluster/IClusterFactory.cs @@ -14,7 +14,7 @@ public interface IClusterFactory /// /// Create checkpoint manager /// - DeviceLogCommitCheckpointManager CreateCheckpointManager(INamedDeviceFactory deviceFactory, ICheckpointNamingScheme checkpointNamingScheme, bool isMainStore, ILogger logger = default); + DeviceLogCommitCheckpointManager CreateCheckpointManager(INamedDeviceFactory deviceFactory, ICheckpointNamingScheme checkpointNamingScheme, bool isMainStore); /// /// Create cluster provider From 78fdc982fa9a01ebc8a9f3d1bdb1e1d1b0168f66 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Tue, 21 Jan 2025 00:32:22 +0100 Subject: [PATCH 06/16] move store wrapper to di --- libs/host/GarnetApplicationBuilder.cs | 187 +++++++++++++++++++++--- libs/host/GarnetServer.cs | 197 +------------------------- 2 files changed, 172 insertions(+), 212 deletions(-) diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index e6c3efa630..a2e643d07d 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Reflection; using Garnet.cluster; using Garnet.common; using Garnet.server; @@ -11,17 +12,35 @@ using Microsoft.Extensions.Diagnostics.Metrics; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Tsavorite.core; namespace Garnet.host; +using MainStoreAllocator = + SpanByteAllocator>; +using MainStoreFunctions = StoreFunctions; +using ObjectStoreAllocator = + GenericAllocator>>; +using ObjectStoreFunctions = + StoreFunctions>; + public class GarnetApplicationBuilder : IHostApplicationBuilder { private readonly HostApplicationBuilder hostApplicationBuilder; + /// + /// Resp protocol version + /// + private readonly string redisProtocolVersion = "7.2.5"; + + private TsavoriteKV store; + private TsavoriteKV objectStore; + internal GarnetApplicationBuilder(GarnetApplicationOptions options) { var configuration = new ConfigurationManager(); - + configuration.AddEnvironmentVariables(prefix: "GARNET_"); hostApplicationBuilder = new HostApplicationBuilder(new HostApplicationBuilderSettings @@ -39,23 +58,19 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) simpleConsoleFormatterOptions.TimestampFormat = "hh::mm::ss "; }); - var garnetOptions = new GarnetServerOptions - { - Port = 6379, - QuietMode = false - }; + var garnetOptions = new GarnetServerOptions {Port = 6379, QuietMode = false}; hostApplicationBuilder.Services.AddSingleton(); hostApplicationBuilder.Services.AddSingleton(sp => { MemoryLogger initLogger; - + // Set up an initial memory logger to log messages from configuration parser into memory. using (var memLogProvider = new MemoryLoggerProvider()) { initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); } - + if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, out var exitGracefully, initLogger)) { @@ -67,8 +82,8 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) throw new GarnetException( "Encountered an error when initializing Garnet server. Please see log messages above for more details."); - } - + } + var loggerFactory = sp.GetRequiredService(); var logger = loggerFactory.CreateLogger("Garnet"); var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); @@ -76,17 +91,147 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) return new GarnetServerTcp(opts.Address, opts.Port, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, opts.NetworkConnectionLimit, logger); }); - - hostApplicationBuilder.Services.AddHostedService(); - /* - hostApplicationBuilder.Services.AddHostedService(sp => + hostApplicationBuilder.Services.AddSingleton(sp => { + MemoryLogger initLogger; + + // Set up an initial memory logger to log messages from configuration parser into memory. + using (var memLogProvider = new MemoryLoggerProvider()) + { + initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); + } + + if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, + out var exitGracefully, initLogger)) + { + if (exitGracefully) + System.Environment.Exit(0); + + // Flush logs from memory logger + //FlushMemoryLogger(this.initLogger, "ArgParser"); + + throw new GarnetException( + "Encountered an error when initializing Garnet server. Please see log messages above for more details."); + } + + var server = sp.GetRequiredService(); + var loggerFactory = sp.GetRequiredService(); - //return new GarnetServer(garnetOptions, loggerFactory, null, cleanupDir: false); - return new GarnetServer(options.Args, loggerFactory, false); + var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); + var customCommandManager = new CustomCommandManager(); + var logger = loggerFactory.CreateLogger("StoreWrapper"); + var clusterFactory = opts.EnableCluster ? sp.GetRequiredService() : null; + + var version = GetVersion(); + + static string GetVersion() + { + var Version = Assembly.GetExecutingAssembly().GetName().Version; + return $"{Version.Major}.{Version.Minor}.{Version.Build}"; + } + + CreateMainStore(clusterFactory, out var checkpointDir); + CreateObjectStore(clusterFactory, customCommandManager, checkpointDir, out var objectStoreSizeTracker); + + void CreateMainStore(IClusterFactory clusterFactory, out string checkpointDir) + { + var kvSettings = opts.GetSettings(loggerFactory, out var logFactory); + + checkpointDir = opts.CheckpointDir ?? opts.LogDir; + + // Run checkpoint on its own thread to control p99 + kvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; + kvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; + + var checkpointFactory = opts.DeviceFactoryCreator(); + if (opts.EnableCluster) + { + kvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager(checkpointFactory, + new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), isMainStore: true); + } + else + { + kvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(checkpointFactory, + new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), removeOutdated: true); + } + + store = new(kvSettings + , StoreFunctions.Create() + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + } + + void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandManager customCommandManager, + string CheckpointDir, out CacheSizeTracker objectStoreSizeTracker) + { + objectStoreSizeTracker = null; + if (!opts.DisableObjects) + { + var objKvSettings = opts.GetObjectStoreSettings(loggerFactory?.CreateLogger("TsavoriteKV [obj]"), + out var objHeapMemorySize, out var objReadCacheHeapMemorySize); + + // Run checkpoint on its own thread to control p99 + objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; + objKvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; + + if (opts.EnableCluster) + objKvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager( + opts.DeviceFactoryCreator(), + new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), + isMainStore: false); + else + objKvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager( + opts.DeviceFactoryCreator(), + new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), + removeOutdated: true); + + objectStore = new(objKvSettings + , StoreFunctions.Create(new ByteArrayKeyComparer(), + () => new ByteArrayBinaryObjectSerializer(), + () => new GarnetObjectSerializer(customCommandManager)) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + + if (objHeapMemorySize > 0 || objReadCacheHeapMemorySize > 0) + objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objHeapMemorySize, + objReadCacheHeapMemorySize, + loggerFactory); + } + } + + TsavoriteLog appendOnlyFile = null; + + if (opts.EnableAOF) + { + if (opts.MainMemoryReplication && opts.CommitFrequencyMs != -1) + throw new Exception("Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); + + opts.GetAofSettings(out var aofSettings); + var aofDevice = aofSettings.LogDevice; + appendOnlyFile = new TsavoriteLog(aofSettings, logger: loggerFactory?.CreateLogger("TsavoriteLog [aof]")); + + if (opts.CommitFrequencyMs < 0 && opts.WaitForCommit) + throw new Exception("Cannot use CommitWait with manual commits"); + } + else + { + if (opts.CommitFrequencyMs != 0 || opts.WaitForCommit) + throw new Exception("Cannot use CommitFrequencyMs or CommitWait without EnableAOF"); + } + + if (logger != null) + { + var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); + if (objectStore != null) + configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0) + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); + logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); + } + + return new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, + customCommandManager, appendOnlyFile, opts, clusterFactory: clusterFactory, + loggerFactory: loggerFactory); }); - */ + + hostApplicationBuilder.Services.AddHostedService(); } public GarnetApplication Build() @@ -103,19 +248,19 @@ public void ConfigureContainer(IServiceProviderFactory Properties { get; } - + public IConfigurationManager Configuration => hostApplicationBuilder.Configuration; - + public IHostEnvironment Environment => hostApplicationBuilder.Environment; public ILoggingBuilder Logging => hostApplicationBuilder.Logging; - + public IMetricsBuilder Metrics => hostApplicationBuilder.Metrics; - + public IServiceCollection Services => hostApplicationBuilder.Services; } \ No newline at end of file diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index ebf20d755b..9933a33598 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -49,9 +49,7 @@ static string GetVersion() private INamedDeviceFactory logFactory; private readonly MemoryLogger initLogger; private readonly ILogger logger; - private readonly ILoggerFactory loggerFactory; private readonly bool cleanupDir; - private bool disposeLoggerFactory; private readonly IClusterFactory clusterFactory; /// @@ -80,15 +78,16 @@ static string GetVersion() public StoreApi Store; public GarnetServer( - ILogger logger, - ILoggerFactory loggerFactory, + ILogger logger, + ILoggerFactory loggerFactory, IClusterFactory clusterFactory, - GarnetServerTcp garnetServerTcp) + GarnetServerTcp garnetServerTcp, + StoreWrapper storeWrapper) { this.logger = logger; - this.loggerFactory = loggerFactory; this.clusterFactory = clusterFactory; this.server = garnetServerTcp; + this.storeWrapper = storeWrapper; Trace.Listeners.Add(new ConsoleTraceListener()); @@ -116,89 +115,6 @@ public GarnetServer( this.InitializeServer(); } - /* - /// - /// Create Garnet Server instance using specified command line arguments; use Start to start the server. - /// - /// Command line arguments - /// Logger factory - /// Clean up directory. - /// Override for custom authentication settings. - public GarnetServer(string[] commandLineArgs, ILoggerFactory loggerFactory = null, bool cleanupDir = false, IAuthenticationSettings authenticationSettingsOverride = null) - { - Trace.Listeners.Add(new ConsoleTraceListener()); - - // Set up an initial memory logger to log messages from configuration parser into memory. - using (var memLogProvider = new MemoryLoggerProvider()) - { - this.initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); - } - - if (!ServerSettingsManager.TryParseCommandLineArguments(commandLineArgs, out var serverSettings, out _, out var exitGracefully, this.initLogger)) - { - if (exitGracefully) - Environment.Exit(0); - - // Flush logs from memory logger - FlushMemoryLogger(this.initLogger, "ArgParser", loggerFactory); - - throw new GarnetException("Encountered an error when initializing Garnet server. Please see log messages above for more details."); - } - - if (loggerFactory == null) - { - // If the main logger factory is created by GarnetServer, it should be disposed when GarnetServer is disposed - disposeLoggerFactory = true; - } - else - { - this.initLogger.LogWarning( - $"Received an external ILoggerFactory object. The following configuration options are ignored: {nameof(serverSettings.FileLogger)}, {nameof(serverSettings.LogLevel)}, {nameof(serverSettings.DisableConsoleLogger)}."); - } - - // If no logger factory is given, set up main logger factory based on parsed configuration values, - // otherwise use given logger factory. - this.loggerFactory = loggerFactory ?? LoggerFactory.Create(builder => - { - if (!serverSettings.DisableConsoleLogger.GetValueOrDefault()) - { - builder.AddSimpleConsole(options => - { - options.SingleLine = true; - options.TimestampFormat = "hh::mm::ss "; - }); - } - - // Optional: Flush log output to file. - if (serverSettings.FileLogger != null) - builder.AddFile(serverSettings.FileLogger); - builder.SetMinimumLevel(serverSettings.LogLevel); - }); - - // Assign values to GarnetServerOptions - this.opts = serverSettings.GetServerOptions(this.loggerFactory.CreateLogger("Options")); - this.opts.AuthSettings = authenticationSettingsOverride ?? this.opts.AuthSettings; - this.cleanupDir = cleanupDir; - this.InitializeServer(); - } - - /// - /// Create Garnet Server instance using GarnetServerOptions instance; use Start to start the server. - /// - /// Server options - /// Logger factory - /// The IGarnetServer to use. If none is provided, will use a GarnetServerTcp. - /// Whether to clean up data folders on dispose - public GarnetServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, IGarnetServer server = null, bool cleanupDir = false) - { - this.server = server; - this.opts = opts; - this.loggerFactory = loggerFactory; - this.cleanupDir = cleanupDir; - this.InitializeServer(); - } - */ - private void InitializeServer() { Debug.Assert(opts != null); @@ -216,9 +132,7 @@ private void InitializeServer() '.' {normal}"); } - - var clusterFactory = opts.EnableCluster ? this.clusterFactory : null; - + using (logger.BeginScope("GarnetServer")) { logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", version, IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.Port); @@ -237,27 +151,11 @@ private void InitializeServer() if (!setMax && !ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads)) throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.ThreadPoolMaxThreads}"); - CreateMainStore(clusterFactory, out var checkpointDir); - CreateObjectStore(clusterFactory, customCommandManager, checkpointDir, out var objectStoreSizeTracker); - if (!opts.DisablePubSub) subscribeBroker = new SubscribeBroker>(new SpanByteKeySerializer(), null, opts.PubSubPageSizeBytes(), opts.SubscriberRefreshFrequencyMs, true); - CreateAOF(); - logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled"); - if (logger != null) - { - var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); - if (objectStore != null) - configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0) + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); - logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); - } - - storeWrapper = new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, - customCommandManager, appendOnlyFile, opts, clusterFactory: clusterFactory, loggerFactory: loggerFactory); - // Create session provider for Garnet Provider = new GarnetProvider(storeWrapper, subscribeBroker); @@ -296,87 +194,6 @@ private void LoadModules(CustomCommandManager customCommandManager) } } - private void CreateMainStore(IClusterFactory clusterFactory, out string checkpointDir) - { - kvSettings = opts.GetSettings(loggerFactory, out logFactory); - - checkpointDir = opts.CheckpointDir ?? opts.LogDir; - - // Run checkpoint on its own thread to control p99 - kvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; - kvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; - - var checkpointFactory = opts.DeviceFactoryCreator(); - if (opts.EnableCluster) - { - kvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager(checkpointFactory, - new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), isMainStore: true); - } - else - { - kvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(checkpointFactory, - new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), removeOutdated: true); - } - - store = new(kvSettings - , StoreFunctions.Create() - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - } - - private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandManager customCommandManager, string CheckpointDir, out CacheSizeTracker objectStoreSizeTracker) - { - objectStoreSizeTracker = null; - if (!opts.DisableObjects) - { - objKvSettings = opts.GetObjectStoreSettings(this.loggerFactory?.CreateLogger("TsavoriteKV [obj]"), - out var objHeapMemorySize, out var objReadCacheHeapMemorySize); - - // Run checkpoint on its own thread to control p99 - objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; - objKvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; - - if (opts.EnableCluster) - objKvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager( - opts.DeviceFactoryCreator(), - new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), - isMainStore: false); - else - objKvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(opts.DeviceFactoryCreator(), - new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), - removeOutdated: true); - - objectStore = new(objKvSettings - , StoreFunctions.Create(new ByteArrayKeyComparer(), - () => new ByteArrayBinaryObjectSerializer(), - () => new GarnetObjectSerializer(customCommandManager)) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - - if (objHeapMemorySize > 0 || objReadCacheHeapMemorySize > 0) - objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objHeapMemorySize, objReadCacheHeapMemorySize, - this.loggerFactory); - } - } - - private void CreateAOF() - { - if (opts.EnableAOF) - { - if (opts.MainMemoryReplication && opts.CommitFrequencyMs != -1) - throw new Exception("Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); - - opts.GetAofSettings(out var aofSettings); - aofDevice = aofSettings.LogDevice; - appendOnlyFile = new TsavoriteLog(aofSettings, logger: this.loggerFactory?.CreateLogger("TsavoriteLog [aof]")); - - if (opts.CommitFrequencyMs < 0 && opts.WaitForCommit) - throw new Exception("Cannot use CommitWait with manual commits"); - return; - } - - if (opts.CommitFrequencyMs != 0 || opts.WaitForCommit) - throw new Exception("Cannot use CommitFrequencyMs or CommitWait without EnableAOF"); - } - /// /// Start server instance /// @@ -434,8 +251,6 @@ private void InternalDispose() objKvSettings.ObjectLogDevice?.Dispose(); } opts.AuthSettings?.Dispose(); - if (disposeLoggerFactory) - loggerFactory?.Dispose(); } /// From 9a9da514f4e068fcc184b52a7e6a41e8ce7ee35f Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Tue, 21 Jan 2025 02:30:16 +0100 Subject: [PATCH 07/16] start refactor store creation --- libs/host/GarnetApplicationBuilder.cs | 196 +++++++++----------------- libs/host/GarnetServer.cs | 63 +-------- libs/host/StoreFactory.cs | 103 ++++++++++++++ 3 files changed, 172 insertions(+), 190 deletions(-) create mode 100644 libs/host/StoreFactory.cs diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index a2e643d07d..39dcd2c478 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -3,7 +3,9 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Reflection; +using System.Text; using Garnet.cluster; using Garnet.common; using Garnet.server; @@ -16,15 +18,6 @@ namespace Garnet.host; -using MainStoreAllocator = - SpanByteAllocator>; -using MainStoreFunctions = StoreFunctions; -using ObjectStoreAllocator = - GenericAllocator>>; -using ObjectStoreFunctions = - StoreFunctions>; - public class GarnetApplicationBuilder : IHostApplicationBuilder { private readonly HostApplicationBuilder hostApplicationBuilder; @@ -34,8 +27,6 @@ public class GarnetApplicationBuilder : IHostApplicationBuilder /// private readonly string redisProtocolVersion = "7.2.5"; - private TsavoriteKV store; - private TsavoriteKV objectStore; internal GarnetApplicationBuilder(GarnetApplicationOptions options) { @@ -58,32 +49,31 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) simpleConsoleFormatterOptions.TimestampFormat = "hh::mm::ss "; }); - var garnetOptions = new GarnetServerOptions {Port = 6379, QuietMode = false}; - hostApplicationBuilder.Services.AddSingleton(); - hostApplicationBuilder.Services.AddSingleton(sp => - { - MemoryLogger initLogger; - - // Set up an initial memory logger to log messages from configuration parser into memory. - using (var memLogProvider = new MemoryLoggerProvider()) - { - initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); - } - - if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, - out var exitGracefully, initLogger)) - { - if (exitGracefully) - System.Environment.Exit(0); + + MemoryLogger initLogger; - // Flush logs from memory logger - //FlushMemoryLogger(this.initLogger, "ArgParser"); + // Set up an initial memory logger to log messages from configuration parser into memory. + using (var memLogProvider = new MemoryLoggerProvider()) + { + initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); + } + + if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, + out var exitGracefully, initLogger)) + { + if (exitGracefully) + System.Environment.Exit(0); - throw new GarnetException( - "Encountered an error when initializing Garnet server. Please see log messages above for more details."); - } + // Flush logs from memory logger + //FlushMemoryLogger(this.initLogger, "ArgParser"); + throw new GarnetException( + "Encountered an error when initializing Garnet server. Please see log messages above for more details."); + } + + hostApplicationBuilder.Services.AddSingleton(sp => + { var loggerFactory = sp.GetRequiredService(); var logger = loggerFactory.CreateLogger("Garnet"); var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); @@ -92,112 +82,30 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) opts.NetworkConnectionLimit, logger); }); - hostApplicationBuilder.Services.AddSingleton(sp => + hostApplicationBuilder.Services.AddSingleton(sp => { - MemoryLogger initLogger; - - // Set up an initial memory logger to log messages from configuration parser into memory. - using (var memLogProvider = new MemoryLoggerProvider()) - { - initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); - } + var loggerFactory = sp.GetRequiredService(); + var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); + var clusterFactory = opts.EnableCluster ? sp.GetRequiredService() : null; - if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, - out var exitGracefully, initLogger)) - { - if (exitGracefully) - System.Environment.Exit(0); - - // Flush logs from memory logger - //FlushMemoryLogger(this.initLogger, "ArgParser"); + return new StoreFactory(clusterFactory, opts, loggerFactory); + }); - throw new GarnetException( - "Encountered an error when initializing Garnet server. Please see log messages above for more details."); - } - + hostApplicationBuilder.Services.AddSingleton(sp => + { var server = sp.GetRequiredService(); - var loggerFactory = sp.GetRequiredService(); var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); var customCommandManager = new CustomCommandManager(); var logger = loggerFactory.CreateLogger("StoreWrapper"); var clusterFactory = opts.EnableCluster ? sp.GetRequiredService() : null; + var storeFactory = sp.GetRequiredService(); var version = GetVersion(); - - static string GetVersion() - { - var Version = Assembly.GetExecutingAssembly().GetName().Version; - return $"{Version.Major}.{Version.Minor}.{Version.Build}"; - } - CreateMainStore(clusterFactory, out var checkpointDir); - CreateObjectStore(clusterFactory, customCommandManager, checkpointDir, out var objectStoreSizeTracker); - - void CreateMainStore(IClusterFactory clusterFactory, out string checkpointDir) - { - var kvSettings = opts.GetSettings(loggerFactory, out var logFactory); - - checkpointDir = opts.CheckpointDir ?? opts.LogDir; - - // Run checkpoint on its own thread to control p99 - kvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; - kvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; - - var checkpointFactory = opts.DeviceFactoryCreator(); - if (opts.EnableCluster) - { - kvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager(checkpointFactory, - new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), isMainStore: true); - } - else - { - kvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(checkpointFactory, - new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), removeOutdated: true); - } - - store = new(kvSettings - , StoreFunctions.Create() - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - } - - void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandManager customCommandManager, - string CheckpointDir, out CacheSizeTracker objectStoreSizeTracker) - { - objectStoreSizeTracker = null; - if (!opts.DisableObjects) - { - var objKvSettings = opts.GetObjectStoreSettings(loggerFactory?.CreateLogger("TsavoriteKV [obj]"), - out var objHeapMemorySize, out var objReadCacheHeapMemorySize); - - // Run checkpoint on its own thread to control p99 - objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; - objKvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; - - if (opts.EnableCluster) - objKvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager( - opts.DeviceFactoryCreator(), - new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), - isMainStore: false); - else - objKvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager( - opts.DeviceFactoryCreator(), - new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), - removeOutdated: true); - - objectStore = new(objKvSettings - , StoreFunctions.Create(new ByteArrayKeyComparer(), - () => new ByteArrayBinaryObjectSerializer(), - () => new GarnetObjectSerializer(customCommandManager)) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - - if (objHeapMemorySize > 0 || objReadCacheHeapMemorySize > 0) - objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objHeapMemorySize, - objReadCacheHeapMemorySize, - loggerFactory); - } - } - + var store= storeFactory.CreateMainStore(out var checkpointDir); + var objectStore = storeFactory.CreateObjectStore(customCommandManager, checkpointDir, out var objectStoreSizeTracker); + TsavoriteLog appendOnlyFile = null; if (opts.EnableAOF) @@ -206,7 +114,6 @@ void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandManager cust throw new Exception("Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); opts.GetAofSettings(out var aofSettings); - var aofDevice = aofSettings.LogDevice; appendOnlyFile = new TsavoriteLog(aofSettings, logger: loggerFactory?.CreateLogger("TsavoriteLog [aof]")); if (opts.CommitFrequencyMs < 0 && opts.WaitForCommit) @@ -225,6 +132,8 @@ void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandManager cust configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0) + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); } + + LoadModules(customCommandManager, opts, logger); return new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, customCommandManager, appendOnlyFile, opts, clusterFactory: clusterFactory, @@ -263,4 +172,35 @@ public IMetricsBuilder Metrics public IServiceCollection Services => hostApplicationBuilder.Services; + + + void LoadModules(CustomCommandManager customCommandManager, GarnetServerOptions opts, ILogger logger) + { + if (opts.LoadModuleCS == null) + return; + + foreach (var moduleCS in opts.LoadModuleCS) + { + var moduleCSData = moduleCS.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (moduleCSData.Length < 1) + continue; + + var modulePath = moduleCSData[0]; + var moduleArgs = moduleCSData.Length > 1 ? moduleCSData.Skip(1).ToArray() : []; + if (ModuleUtils.LoadAssemblies([modulePath], null, true, out var loadedAssemblies, out var errorMsg)) + { + ModuleRegistrar.Instance.LoadModule(customCommandManager, loadedAssemblies.ToList()[0], moduleArgs, logger, out errorMsg); + } + else + { + logger?.LogError("Module {0} failed to load with error {1}", modulePath, Encoding.UTF8.GetString(errorMsg)); + } + } + } + + private static string GetVersion() + { + var Version = Assembly.GetExecutingAssembly().GetName().Version; + return $"{Version.Major}.{Version.Minor}.{Version.Build}"; + } } \ No newline at end of file diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 9933a33598..b048729250 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -3,9 +3,7 @@ using System; using System.Diagnostics; -using System.Linq; using System.Reflection; -using System.Text; using System.Threading; using System.Threading.Tasks; using Garnet.common; @@ -16,12 +14,6 @@ using Tsavorite.core; namespace Garnet; - -using MainStoreAllocator = SpanByteAllocator>; -using MainStoreFunctions = StoreFunctions; - -using ObjectStoreAllocator = GenericAllocator>>; -using ObjectStoreFunctions = StoreFunctions>; /// /// Implementation Garnet server @@ -39,17 +31,9 @@ static string GetVersion() private readonly GarnetServerOptions opts; private IGarnetServer server; - private TsavoriteKV store; - private TsavoriteKV objectStore; - private IDevice aofDevice; - private TsavoriteLog appendOnlyFile; private SubscribeBroker> subscribeBroker; - private KVSettings kvSettings; - private KVSettings objKvSettings; - private INamedDeviceFactory logFactory; private readonly MemoryLogger initLogger; private readonly ILogger logger; - private readonly bool cleanupDir; private readonly IClusterFactory clusterFactory; /// @@ -57,11 +41,6 @@ static string GetVersion() /// protected StoreWrapper storeWrapper; - /// - /// Resp protocol version - /// - readonly string redisProtocolVersion = "7.2.5"; - /// /// Metrics API /// @@ -111,7 +90,6 @@ public GarnetServer( // Assign values to GarnetServerOptions this.opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); this.opts.AuthSettings = this.opts.AuthSettings; - this.cleanupDir = cleanupDir; this.InitializeServer(); } @@ -140,8 +118,6 @@ private void InitializeServer() // Flush initialization logs from memory logger FlushMemoryLogger(this.initLogger, "ArgParser"); - var customCommandManager = new CustomCommandManager(); - var setMax = opts.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads); if (opts.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.ThreadPoolMinThreads, opts.ThreadPoolMinThreads)) @@ -165,32 +141,6 @@ private void InitializeServer() Store = new StoreApi(storeWrapper); server.Register(WireFormat.ASCII, Provider); - - LoadModules(customCommandManager); - } - } - - private void LoadModules(CustomCommandManager customCommandManager) - { - if (opts.LoadModuleCS == null) - return; - - foreach (var moduleCS in opts.LoadModuleCS) - { - var moduleCSData = moduleCS.Split(' ', StringSplitOptions.RemoveEmptyEntries); - if (moduleCSData.Length < 1) - continue; - - var modulePath = moduleCSData[0]; - var moduleArgs = moduleCSData.Length > 1 ? moduleCSData.Skip(1).ToArray() : []; - if (ModuleUtils.LoadAssemblies([modulePath], null, true, out var loadedAssemblies, out var errorMsg)) - { - ModuleRegistrar.Instance.LoadModule(customCommandManager, loadedAssemblies.ToList()[0], moduleArgs, logger, out errorMsg); - } - else - { - logger?.LogError("Module {0} failed to load with error {1}", modulePath, Encoding.UTF8.GetString(errorMsg)); - } } } @@ -211,7 +161,7 @@ public void Start() /// public void Dispose() { - Dispose(cleanupDir); + Dispose(false); } /// @@ -223,7 +173,6 @@ public void Dispose(bool deleteDir = true) InternalDispose(); if (deleteDir) { - logFactory?.Delete(new FileDescriptor { directoryName = "" }); if (opts.CheckpointDir != opts.LogDir && !string.IsNullOrEmpty(opts.CheckpointDir)) { var ckptdir = opts.DeviceFactoryCreator(); @@ -240,16 +189,6 @@ private void InternalDispose() server.Dispose(); subscribeBroker?.Dispose(); - store.Dispose(); - appendOnlyFile?.Dispose(); - aofDevice?.Dispose(); - kvSettings.LogDevice?.Dispose(); - if (!opts.DisableObjects) - { - objectStore.Dispose(); - objKvSettings.LogDevice?.Dispose(); - objKvSettings.ObjectLogDevice?.Dispose(); - } opts.AuthSettings?.Dispose(); } diff --git a/libs/host/StoreFactory.cs b/libs/host/StoreFactory.cs new file mode 100644 index 0000000000..2aa65065a7 --- /dev/null +++ b/libs/host/StoreFactory.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.server; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.host; + +using MainStoreAllocator = + SpanByteAllocator>; +using MainStoreFunctions = StoreFunctions; +using ObjectStoreAllocator = + GenericAllocator>>; +using ObjectStoreFunctions = + StoreFunctions>; + +public class StoreFactory +{ + private readonly IClusterFactory clusterFactory; + private readonly GarnetServerOptions opts; + private readonly ILoggerFactory loggerFactory; + + public StoreFactory(IClusterFactory clusterFactory, GarnetServerOptions opts, ILoggerFactory loggerFactory) + { + this.clusterFactory = clusterFactory; + this.opts = opts; + this.loggerFactory = loggerFactory; + } + + public TsavoriteKV CreateObjectStore(CustomCommandManager customCommandManager, + string CheckpointDir, out CacheSizeTracker objectStoreSizeTracker) + { + TsavoriteKV objectStore = null; + + objectStoreSizeTracker = null; + if (!opts.DisableObjects) + { + var objKvSettings = opts.GetObjectStoreSettings(loggerFactory?.CreateLogger("TsavoriteKV [obj]"), + out var objHeapMemorySize, out var objReadCacheHeapMemorySize); + + // Run checkpoint on its own thread to control p99 + objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; + objKvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; + + if (opts.EnableCluster) + objKvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager( + opts.DeviceFactoryCreator(), + new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), + isMainStore: false); + else + objKvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager( + opts.DeviceFactoryCreator(), + new DefaultCheckpointNamingScheme(CheckpointDir + "/ObjectStore/checkpoints"), + removeOutdated: true); + + objectStore = new(objKvSettings + , StoreFunctions.Create(new ByteArrayKeyComparer(), + () => new ByteArrayBinaryObjectSerializer(), + () => new GarnetObjectSerializer(customCommandManager)) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + + if (objHeapMemorySize > 0 || objReadCacheHeapMemorySize > 0) + objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objHeapMemorySize, + objReadCacheHeapMemorySize, + loggerFactory); + } + + return objectStore; + } + + public TsavoriteKV CreateMainStore(out string checkpointDir) + { + TsavoriteKV store = null; + + var kvSettings = opts.GetSettings(loggerFactory, out var logFactory); + + checkpointDir = opts.CheckpointDir ?? opts.LogDir; + + // Run checkpoint on its own thread to control p99 + kvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; + kvSettings.CheckpointVersionSwitchBarrier = opts.EnableCluster; + + var checkpointFactory = opts.DeviceFactoryCreator(); + if (opts.EnableCluster) + { + kvSettings.CheckpointManager = clusterFactory.CreateCheckpointManager(checkpointFactory, + new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), isMainStore: true); + } + else + { + kvSettings.CheckpointManager = new DeviceLogCommitCheckpointManager(checkpointFactory, + new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), removeOutdated: true); + } + + store = new(kvSettings + , StoreFunctions.Create() + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + + return store; + } +} \ No newline at end of file From 5a162cd6e45efbc24113f876dd3c5ce51aee8b16 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Tue, 21 Jan 2025 02:44:16 +0100 Subject: [PATCH 08/16] refactor GarnetServerTcp --- libs/host/GarnetApplicationBuilder.cs | 71 ++++++++++++++------------ libs/host/StoreFactory.cs | 24 ++++----- libs/server/Servers/GarnetServerTcp.cs | 18 +++---- 3 files changed, 58 insertions(+), 55 deletions(-) diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 39dcd2c478..8809e08e04 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -27,7 +27,6 @@ public class GarnetApplicationBuilder : IHostApplicationBuilder /// private readonly string redisProtocolVersion = "7.2.5"; - internal GarnetApplicationBuilder(GarnetApplicationOptions options) { var configuration = new ConfigurationManager(); @@ -50,7 +49,7 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) }); hostApplicationBuilder.Services.AddSingleton(); - + MemoryLogger initLogger; // Set up an initial memory logger to log messages from configuration parser into memory. @@ -58,7 +57,7 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) { initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); } - + if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, out var exitGracefully, initLogger)) { @@ -71,50 +70,54 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) throw new GarnetException( "Encountered an error when initializing Garnet server. Please see log messages above for more details."); } - + hostApplicationBuilder.Services.AddSingleton(sp => { var loggerFactory = sp.GetRequiredService(); - var logger = loggerFactory.CreateLogger("Garnet"); + var logger = sp.GetRequiredService>(); var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); - return new GarnetServerTcp(opts.Address, opts.Port, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, - opts.NetworkConnectionLimit, logger); + return new GarnetServerTcp(opts, logger); }); - hostApplicationBuilder.Services.AddSingleton(sp => + hostApplicationBuilder.Services.AddSingleton(sp => { var loggerFactory = sp.GetRequiredService(); var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); var clusterFactory = opts.EnableCluster ? sp.GetRequiredService() : null; - - return new StoreFactory(clusterFactory, opts, loggerFactory); + var customCommandManager = sp.GetRequiredService(); + + return new StoreFactory(clusterFactory, opts, loggerFactory, customCommandManager); }); + hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddSingleton(sp => { var server = sp.GetRequiredService(); var loggerFactory = sp.GetRequiredService(); var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); - var customCommandManager = new CustomCommandManager(); + var customCommandManager = sp.GetRequiredService(); var logger = loggerFactory.CreateLogger("StoreWrapper"); var clusterFactory = opts.EnableCluster ? sp.GetRequiredService() : null; var storeFactory = sp.GetRequiredService(); var version = GetVersion(); - - var store= storeFactory.CreateMainStore(out var checkpointDir); - var objectStore = storeFactory.CreateObjectStore(customCommandManager, checkpointDir, out var objectStoreSizeTracker); - + + var store = storeFactory.CreateMainStore(out var checkpointDir); + var objectStore = storeFactory.CreateObjectStore(checkpointDir, out var objectStoreSizeTracker); + TsavoriteLog appendOnlyFile = null; - + if (opts.EnableAOF) { if (opts.MainMemoryReplication && opts.CommitFrequencyMs != -1) - throw new Exception("Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); + throw new Exception( + "Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); opts.GetAofSettings(out var aofSettings); - appendOnlyFile = new TsavoriteLog(aofSettings, logger: loggerFactory?.CreateLogger("TsavoriteLog [aof]")); + appendOnlyFile = + new TsavoriteLog(aofSettings, logger: loggerFactory?.CreateLogger("TsavoriteLog [aof]")); if (opts.CommitFrequencyMs < 0 && opts.WaitForCommit) throw new Exception("Cannot use CommitWait with manual commits"); @@ -124,15 +127,17 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) if (opts.CommitFrequencyMs != 0 || opts.WaitForCommit) throw new Exception("Cannot use CommitFrequencyMs or CommitWait without EnableAOF"); } - - if (logger != null) - { - var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); - if (objectStore != null) - configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0) + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); - logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); - } - + + var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); + if (objectStore != null) + configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + + (objectStoreSizeTracker?.TargetSize ?? 0) + + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); + logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); + LoadModules(customCommandManager, opts, logger); return new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, @@ -172,8 +177,8 @@ public IMetricsBuilder Metrics public IServiceCollection Services => hostApplicationBuilder.Services; - - + + void LoadModules(CustomCommandManager customCommandManager, GarnetServerOptions opts, ILogger logger) { if (opts.LoadModuleCS == null) @@ -189,15 +194,17 @@ void LoadModules(CustomCommandManager customCommandManager, GarnetServerOptions var moduleArgs = moduleCSData.Length > 1 ? moduleCSData.Skip(1).ToArray() : []; if (ModuleUtils.LoadAssemblies([modulePath], null, true, out var loadedAssemblies, out var errorMsg)) { - ModuleRegistrar.Instance.LoadModule(customCommandManager, loadedAssemblies.ToList()[0], moduleArgs, logger, out errorMsg); + ModuleRegistrar.Instance.LoadModule(customCommandManager, loadedAssemblies.ToList()[0], moduleArgs, + logger, out errorMsg); } else { - logger?.LogError("Module {0} failed to load with error {1}", modulePath, Encoding.UTF8.GetString(errorMsg)); + logger?.LogError("Module {0} failed to load with error {1}", modulePath, + Encoding.UTF8.GetString(errorMsg)); } } } - + private static string GetVersion() { var Version = Assembly.GetExecutingAssembly().GetName().Version; diff --git a/libs/host/StoreFactory.cs b/libs/host/StoreFactory.cs index 2aa65065a7..ce050eef47 100644 --- a/libs/host/StoreFactory.cs +++ b/libs/host/StoreFactory.cs @@ -21,19 +21,22 @@ public class StoreFactory private readonly IClusterFactory clusterFactory; private readonly GarnetServerOptions opts; private readonly ILoggerFactory loggerFactory; - - public StoreFactory(IClusterFactory clusterFactory, GarnetServerOptions opts, ILoggerFactory loggerFactory) + private readonly CustomCommandManager customCommandManager; + + public StoreFactory(IClusterFactory clusterFactory, GarnetServerOptions opts, ILoggerFactory loggerFactory, + CustomCommandManager customCommandManager) { this.clusterFactory = clusterFactory; this.opts = opts; this.loggerFactory = loggerFactory; + this.customCommandManager = customCommandManager; } - - public TsavoriteKV CreateObjectStore(CustomCommandManager customCommandManager, + + public TsavoriteKV CreateObjectStore( string CheckpointDir, out CacheSizeTracker objectStoreSizeTracker) { TsavoriteKV objectStore = null; - + objectStoreSizeTracker = null; if (!opts.DisableObjects) { @@ -69,11 +72,10 @@ public TsavoriteKV CreateMainStore(out string checkpointDir) + + public TsavoriteKV CreateMainStore( + out string checkpointDir) { - TsavoriteKV store = null; - var kvSettings = opts.GetSettings(loggerFactory, out var logFactory); checkpointDir = opts.CheckpointDir ?? opts.LogDir; @@ -94,10 +96,8 @@ public TsavoriteKV C new DefaultCheckpointNamingScheme(checkpointDir + "/Store/checkpoints"), removeOutdated: true); } - store = new(kvSettings + return new(kvSettings , StoreFunctions.Create() , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - - return store; } } \ No newline at end of file diff --git a/libs/server/Servers/GarnetServerTcp.cs b/libs/server/Servers/GarnetServerTcp.cs index 09a1c4fd71..caba315e7b 100644 --- a/libs/server/Servers/GarnetServerTcp.cs +++ b/libs/server/Servers/GarnetServerTcp.cs @@ -25,7 +25,7 @@ public class GarnetServerTcp : GarnetServerBase, IServerHook readonly NetworkBufferSettings networkBufferSettings; readonly LimitedFixedBufferPool networkPool; readonly int networkConnectionLimit; - + public IPEndPoint GetEndPoint { get @@ -64,18 +64,14 @@ public IEnumerable ActiveClusterSessions() /// /// Constructor for server /// - /// - /// - /// - /// - /// + /// /// - public GarnetServerTcp(string address, int port, int networkBufferSize = default, IGarnetTlsOptions tlsOptions = null, int networkSendThrottleMax = 8, int networkConnectionLimit = -1, ILogger logger = null) - : base(address, port, networkBufferSize, logger) + public GarnetServerTcp(GarnetServerOptions opts, ILogger logger) + : base(opts.Address, opts.Port, 0, logger) { - this.networkConnectionLimit = networkConnectionLimit; - this.tlsOptions = tlsOptions; - this.networkSendThrottleMax = networkSendThrottleMax; + this.networkConnectionLimit = opts.NetworkConnectionLimit; + this.tlsOptions = opts.TlsOptions; + this.networkSendThrottleMax = opts.NetworkSendThrottleMax; var serverBufferSize = BufferSizeUtils.ServerBufferSize(new MaxSizeSettings()); this.networkBufferSettings = new NetworkBufferSettings(serverBufferSize, serverBufferSize); this.networkPool = networkBufferSettings.CreateBufferPool(logger: logger); From e8cc1954c5c1aadb30dc474a7d8576280c5b25d4 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Tue, 21 Jan 2025 03:05:10 +0100 Subject: [PATCH 09/16] implement options pattern --- libs/host/GarnetApplicationBuilder.cs | 40 +++++++++++++++----------- libs/host/StoreFactory.cs | 5 ++-- libs/server/Servers/GarnetServerTcp.cs | 11 +++---- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 8809e08e04..37848e32cf 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -14,6 +14,7 @@ using Microsoft.Extensions.Diagnostics.Metrics; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Tsavorite.core; namespace Garnet.host; @@ -48,6 +49,8 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) simpleConsoleFormatterOptions.TimestampFormat = "hh::mm::ss "; }); + hostApplicationBuilder.Services.AddOptions(); + hostApplicationBuilder.Services.AddSingleton(); MemoryLogger initLogger; @@ -70,24 +73,27 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) throw new GarnetException( "Encountered an error when initializing Garnet server. Please see log messages above for more details."); } - + + var garnetServerOptions = serverSettings.GetServerOptions(null); + var garnetServerOptionsWrapped = Microsoft.Extensions.Options.Options.Create(garnetServerOptions); + hostApplicationBuilder.Services.AddSingleton(garnetServerOptionsWrapped); + hostApplicationBuilder.Services.AddSingleton(sp => { - var loggerFactory = sp.GetRequiredService(); var logger = sp.GetRequiredService>(); - var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); - - return new GarnetServerTcp(opts, logger); + var o = sp.GetRequiredService>(); + + return new GarnetServerTcp(o, logger); }); hostApplicationBuilder.Services.AddSingleton(sp => { var loggerFactory = sp.GetRequiredService(); - var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); - var clusterFactory = opts.EnableCluster ? sp.GetRequiredService() : null; + var o = sp.GetRequiredService>(); + var clusterFactory = o.Value.EnableCluster ? sp.GetRequiredService() : null; var customCommandManager = sp.GetRequiredService(); - return new StoreFactory(clusterFactory, opts, loggerFactory, customCommandManager); + return new StoreFactory(clusterFactory, o, loggerFactory, customCommandManager); }); hostApplicationBuilder.Services.AddSingleton(); @@ -96,10 +102,10 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) { var server = sp.GetRequiredService(); var loggerFactory = sp.GetRequiredService(); - var opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); + var opts = sp.GetRequiredService>(); var customCommandManager = sp.GetRequiredService(); var logger = loggerFactory.CreateLogger("StoreWrapper"); - var clusterFactory = opts.EnableCluster ? sp.GetRequiredService() : null; + var clusterFactory = opts.Value.EnableCluster ? sp.GetRequiredService() : null; var storeFactory = sp.GetRequiredService(); var version = GetVersion(); @@ -109,22 +115,22 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) TsavoriteLog appendOnlyFile = null; - if (opts.EnableAOF) + if (opts.Value.EnableAOF) { - if (opts.MainMemoryReplication && opts.CommitFrequencyMs != -1) + if (opts.Value.MainMemoryReplication && opts.Value.CommitFrequencyMs != -1) throw new Exception( "Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); - opts.GetAofSettings(out var aofSettings); + opts.Value.GetAofSettings(out var aofSettings); appendOnlyFile = new TsavoriteLog(aofSettings, logger: loggerFactory?.CreateLogger("TsavoriteLog [aof]")); - if (opts.CommitFrequencyMs < 0 && opts.WaitForCommit) + if (opts.Value.CommitFrequencyMs < 0 && opts.Value.WaitForCommit) throw new Exception("Cannot use CommitWait with manual commits"); } else { - if (opts.CommitFrequencyMs != 0 || opts.WaitForCommit) + if (opts.Value.CommitFrequencyMs != 0 || opts.Value.WaitForCommit) throw new Exception("Cannot use CommitFrequencyMs or CommitWait without EnableAOF"); } @@ -138,10 +144,10 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); - LoadModules(customCommandManager, opts, logger); + LoadModules(customCommandManager, opts.Value, logger); return new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, - customCommandManager, appendOnlyFile, opts, clusterFactory: clusterFactory, + customCommandManager, appendOnlyFile, opts.Value, clusterFactory: clusterFactory, loggerFactory: loggerFactory); }); diff --git a/libs/host/StoreFactory.cs b/libs/host/StoreFactory.cs index ce050eef47..4ffbe42f3c 100644 --- a/libs/host/StoreFactory.cs +++ b/libs/host/StoreFactory.cs @@ -3,6 +3,7 @@ using Garnet.server; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Tsavorite.core; namespace Garnet.host; @@ -23,11 +24,11 @@ public class StoreFactory private readonly ILoggerFactory loggerFactory; private readonly CustomCommandManager customCommandManager; - public StoreFactory(IClusterFactory clusterFactory, GarnetServerOptions opts, ILoggerFactory loggerFactory, + public StoreFactory(IClusterFactory clusterFactory, IOptions opts, ILoggerFactory loggerFactory, CustomCommandManager customCommandManager) { this.clusterFactory = clusterFactory; - this.opts = opts; + this.opts = opts.Value; this.loggerFactory = loggerFactory; this.customCommandManager = customCommandManager; } diff --git a/libs/server/Servers/GarnetServerTcp.cs b/libs/server/Servers/GarnetServerTcp.cs index caba315e7b..1f950beebd 100644 --- a/libs/server/Servers/GarnetServerTcp.cs +++ b/libs/server/Servers/GarnetServerTcp.cs @@ -10,6 +10,7 @@ using Garnet.networking; using Garnet.server.TLS; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Garnet.server { @@ -66,12 +67,12 @@ public IEnumerable ActiveClusterSessions() /// /// /// - public GarnetServerTcp(GarnetServerOptions opts, ILogger logger) - : base(opts.Address, opts.Port, 0, logger) + public GarnetServerTcp(IOptions opts, ILogger logger) + : base(opts.Value.Address, opts.Value.Port, 0, logger) { - this.networkConnectionLimit = opts.NetworkConnectionLimit; - this.tlsOptions = opts.TlsOptions; - this.networkSendThrottleMax = opts.NetworkSendThrottleMax; + this.networkConnectionLimit = opts.Value.NetworkConnectionLimit; + this.tlsOptions = opts.Value.TlsOptions; + this.networkSendThrottleMax = opts.Value.NetworkSendThrottleMax; var serverBufferSize = BufferSizeUtils.ServerBufferSize(new MaxSizeSettings()); this.networkBufferSettings = new NetworkBufferSettings(serverBufferSize, serverBufferSize); this.networkPool = networkBufferSettings.CreateBufferPool(logger: logger); From bd7d38ead10ec879afd1f88f22d6b74b9ab2b7ce Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Tue, 21 Jan 2025 03:41:35 +0100 Subject: [PATCH 10/16] more refactoring --- .../Embedded/EmbeddedRespServer.cs | 3 +- hosting/Windows/Garnet.worker/Worker.cs | 15 ++----- libs/cluster/Server/ClusterProvider.cs | 13 +++--- .../Server/Migration/MigrateSession.cs | 3 +- libs/host/GarnetApplication.cs | 2 +- libs/host/GarnetApplicationBuilder.cs | 44 +++++++++---------- libs/host/GarnetServer.cs | 8 ++-- libs/host/StoreFactory.cs | 2 +- libs/server/AOF/AofProcessor.cs | 4 +- libs/server/StoreWrapper.cs | 21 ++++----- 10 files changed, 50 insertions(+), 65 deletions(-) diff --git a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs index c94867b102..186e442f9a 100644 --- a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs +++ b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs @@ -20,7 +20,8 @@ internal sealed class EmbeddedRespServer : GarnetServer /// Server options to configure the base GarnetServer instance /// Logger factory to configure the base GarnetServer instance /// Server network - public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, GarnetServerEmbedded server = null) : base(opts, loggerFactory, server) + public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, GarnetServerEmbedded server = null) + : base(opts, loggerFactory, server) { this.garnetServerEmbedded = server; } diff --git a/hosting/Windows/Garnet.worker/Worker.cs b/hosting/Windows/Garnet.worker/Worker.cs index d69adb7e3c..b6de5ea2ec 100644 --- a/hosting/Windows/Garnet.worker/Worker.cs +++ b/hosting/Windows/Garnet.worker/Worker.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Garnet.host; using Microsoft.Extensions.Hosting; namespace Garnet @@ -22,19 +23,11 @@ public Worker(string[] args) protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - try - { - server = new GarnetServer(args); + var builder = GarnetApplication.CreateHostBuilder(args); - // Start the server - server.Start(); + var app = builder.Build(); - await Task.Delay(Timeout.Infinite, stoppingToken).ConfigureAwait(false); - } - catch (Exception ex) - { - Console.WriteLine($"Unable to initialize server due to exception: {ex.Message}"); - } + await app.RunAsync(stoppingToken); } /// diff --git a/libs/cluster/Server/ClusterProvider.cs b/libs/cluster/Server/ClusterProvider.cs index b1e9abc1ee..bc42c86baf 100644 --- a/libs/cluster/Server/ClusterProvider.cs +++ b/libs/cluster/Server/ClusterProvider.cs @@ -31,7 +31,6 @@ public class ClusterProvider : IClusterProvider internal readonly ReplicationManager replicationManager; internal readonly FailoverManager failoverManager; internal readonly MigrationManager migrationManager; - internal readonly ILoggerFactory loggerFactory; internal readonly StoreWrapper storeWrapper; internal readonly GarnetServerOptions serverOptions; internal long GarnetCurrentEpoch = 1; @@ -47,6 +46,8 @@ public class ClusterProvider : IClusterProvider /// public string ClusterPassword => authContainer.ClusterPassword; + public ILogger logger; + /// /// Create new cluster provider /// @@ -54,7 +55,7 @@ public ClusterProvider(StoreWrapper storeWrapper) { this.storeWrapper = storeWrapper; this.serverOptions = storeWrapper.serverOptions; - this.loggerFactory = storeWrapper.loggerFactory; + this.logger = storeWrapper.logger; authContainer = new ClusterAuthContainer { @@ -67,11 +68,11 @@ public ClusterProvider(StoreWrapper storeWrapper) throw new Exception("Gossip sample fraction should be in range [0,100]"); } - this.clusterManager = new ClusterManager(this, logger: loggerFactory?.CreateLogger("ClusterManager")); - this.replicationManager = new ReplicationManager(this, logger: loggerFactory?.CreateLogger("ReplicationManager")); + this.clusterManager = new ClusterManager(this, logger: logger); + this.replicationManager = new ReplicationManager(this, logger: logger); - this.failoverManager = new FailoverManager(this, logger: loggerFactory?.CreateLogger("FailoverManager")); - this.migrationManager = new MigrationManager(this, logger: loggerFactory?.CreateLogger("MigrationManager")); + this.failoverManager = new FailoverManager(this, logger: logger); + this.migrationManager = new MigrationManager(this, logger: logger); } /// diff --git a/libs/cluster/Server/Migration/MigrateSession.cs b/libs/cluster/Server/Migration/MigrateSession.cs index 04612e8360..e8b5648350 100644 --- a/libs/cluster/Server/Migration/MigrateSession.cs +++ b/libs/cluster/Server/Migration/MigrateSession.cs @@ -127,7 +127,8 @@ internal MigrateSession( MigratingKeysWorkingSet keys, TransferOption transferOption) { - this.logger = clusterProvider.loggerFactory.CreateLogger($"MigrateSession - {GetHashCode()}"); ; + //this.logger = clusterProvider.loggerFactory.CreateLogger($"MigrateSession - {GetHashCode()}"); ; + this.logger = clusterProvider.logger; this.clusterSession = clusterSession; this.clusterProvider = clusterProvider; this._targetAddress = _targetAddress; diff --git a/libs/host/GarnetApplication.cs b/libs/host/GarnetApplication.cs index 91a19cdc50..c95eccc849 100644 --- a/libs/host/GarnetApplication.cs +++ b/libs/host/GarnetApplication.cs @@ -30,7 +30,7 @@ public Task StopAsync(CancellationToken cancellationToken = default) public void Dispose() => host.Dispose(); - public void Run(CancellationToken cancellationToken = default) + public void Run() { HostingAbstractionsHostExtensions.Run(this); } diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 37848e32cf..5b5a09a566 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -77,34 +77,17 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) var garnetServerOptions = serverSettings.GetServerOptions(null); var garnetServerOptionsWrapped = Microsoft.Extensions.Options.Options.Create(garnetServerOptions); hostApplicationBuilder.Services.AddSingleton(garnetServerOptionsWrapped); - - hostApplicationBuilder.Services.AddSingleton(sp => - { - var logger = sp.GetRequiredService>(); - var o = sp.GetRequiredService>(); - - return new GarnetServerTcp(o, logger); - }); - - hostApplicationBuilder.Services.AddSingleton(sp => - { - var loggerFactory = sp.GetRequiredService(); - var o = sp.GetRequiredService>(); - var clusterFactory = o.Value.EnableCluster ? sp.GetRequiredService() : null; - var customCommandManager = sp.GetRequiredService(); - - return new StoreFactory(clusterFactory, o, loggerFactory, customCommandManager); - }); + hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddSingleton(); hostApplicationBuilder.Services.AddSingleton(); hostApplicationBuilder.Services.AddSingleton(sp => { var server = sp.GetRequiredService(); - var loggerFactory = sp.GetRequiredService(); var opts = sp.GetRequiredService>(); var customCommandManager = sp.GetRequiredService(); - var logger = loggerFactory.CreateLogger("StoreWrapper"); + var logger = sp.GetRequiredService>(); var clusterFactory = opts.Value.EnableCluster ? sp.GetRequiredService() : null; var storeFactory = sp.GetRequiredService(); @@ -122,8 +105,10 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) "Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); opts.Value.GetAofSettings(out var aofSettings); - appendOnlyFile = - new TsavoriteLog(aofSettings, logger: loggerFactory?.CreateLogger("TsavoriteLog [aof]")); + + var tsavoriteLogLogger = sp.GetRequiredService>(); + + appendOnlyFile = new TsavoriteLog(aofSettings, tsavoriteLogLogger); if (opts.Value.CommitFrequencyMs < 0 && opts.Value.WaitForCommit) throw new Exception("Cannot use CommitWait with manual commits"); @@ -147,11 +132,22 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) LoadModules(customCommandManager, opts.Value, logger); return new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, - customCommandManager, appendOnlyFile, opts.Value, clusterFactory: clusterFactory, - loggerFactory: loggerFactory); + customCommandManager, appendOnlyFile, opts.Value, logger, clusterFactory: clusterFactory); }); hostApplicationBuilder.Services.AddHostedService(); + + /* + hostApplicationBuilder.Services.AddSingleton(sp => + { + var opts = sp.GetRequiredService>(); + var wrapper = sp.GetRequiredService(); + var server = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + + return new GarnetServerMonitor(wrapper, opts.Value, server, logger); + }); + */ } public GarnetApplication Build() diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index b048729250..662d511bbd 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -11,6 +11,7 @@ using Garnet.server; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Tsavorite.core; namespace Garnet; @@ -34,7 +35,6 @@ static string GetVersion() private SubscribeBroker> subscribeBroker; private readonly MemoryLogger initLogger; private readonly ILogger logger; - private readonly IClusterFactory clusterFactory; /// /// Store and associated information used by this Garnet server @@ -57,14 +57,12 @@ static string GetVersion() public StoreApi Store; public GarnetServer( + IOptions opts, ILogger logger, - ILoggerFactory loggerFactory, - IClusterFactory clusterFactory, GarnetServerTcp garnetServerTcp, StoreWrapper storeWrapper) { this.logger = logger; - this.clusterFactory = clusterFactory; this.server = garnetServerTcp; this.storeWrapper = storeWrapper; @@ -88,7 +86,7 @@ public GarnetServer( } // Assign values to GarnetServerOptions - this.opts = serverSettings.GetServerOptions(loggerFactory.CreateLogger("Options")); + this.opts = opts.Value; this.opts.AuthSettings = this.opts.AuthSettings; this.InitializeServer(); } diff --git a/libs/host/StoreFactory.cs b/libs/host/StoreFactory.cs index 4ffbe42f3c..bb6cb01a0e 100644 --- a/libs/host/StoreFactory.cs +++ b/libs/host/StoreFactory.cs @@ -27,7 +27,7 @@ public class StoreFactory public StoreFactory(IClusterFactory clusterFactory, IOptions opts, ILoggerFactory loggerFactory, CustomCommandManager customCommandManager) { - this.clusterFactory = clusterFactory; + this.clusterFactory = opts.Value.EnableCluster ? clusterFactory : null; this.opts = opts.Value; this.loggerFactory = loggerFactory; this.customCommandManager = customCommandManager; diff --git a/libs/server/AOF/AofProcessor.cs b/libs/server/AOF/AofProcessor.cs index 94bf35913c..0818f8eb42 100644 --- a/libs/server/AOF/AofProcessor.cs +++ b/libs/server/AOF/AofProcessor.cs @@ -78,8 +78,8 @@ public AofProcessor( storeWrapper.customCommandManager, recordToAof ? storeWrapper.appendOnlyFile : null, storeWrapper.serverOptions, - accessControlList: storeWrapper.accessControlList, - loggerFactory: storeWrapper.loggerFactory); + storeWrapper.logger, + accessControlList: storeWrapper.accessControlList); this.respServerSession = new RespServerSession(0, networkSender: null, storeWrapper: replayAofStoreWrapper, subscribeBroker: null, authenticator: null, enableScripts: false); diff --git a/libs/server/StoreWrapper.cs b/libs/server/StoreWrapper.cs index 428003f335..c6856f92f9 100644 --- a/libs/server/StoreWrapper.cs +++ b/libs/server/StoreWrapper.cs @@ -71,11 +71,6 @@ public sealed class StoreWrapper internal long lastSaveStoreTailAddress; internal long lastSaveObjectStoreTailAddress; - /// - /// Logger factory - /// - public readonly ILoggerFactory loggerFactory; - internal readonly CollectionItemBroker itemBroker; internal readonly CustomCommandManager customCommandManager; internal readonly GarnetServerMonitor monitor; @@ -88,7 +83,7 @@ public sealed class StoreWrapper /// /// The main logger instance associated with this store. /// - public readonly ILogger logger; + public readonly ILogger logger; internal readonly ILogger sessionLogger; readonly CancellationTokenSource ctsCommit; @@ -124,10 +119,9 @@ public StoreWrapper( CustomCommandManager customCommandManager, TsavoriteLog appendOnlyFile, GarnetServerOptions serverOptions, + ILogger logger, AccessControlList accessControlList = null, - IClusterFactory clusterFactory = null, - ILoggerFactory loggerFactory = null - ) + IClusterFactory clusterFactory = null) { this.version = version; this.redisProtocolVersion = redisProtocolVersion; @@ -139,11 +133,12 @@ public StoreWrapper( this.serverOptions = serverOptions; lastSaveTime = DateTimeOffset.FromUnixTimeSeconds(0); this.customCommandManager = customCommandManager; - this.monitor = serverOptions.MetricsSamplingFrequency > 0 ? new GarnetServerMonitor(this, serverOptions, server, loggerFactory?.CreateLogger("GarnetServerMonitor")) : null; + this.monitor = serverOptions.MetricsSamplingFrequency > 0 + ? new GarnetServerMonitor(this, serverOptions, server, logger) + : null; this.objectStoreSizeTracker = objectStoreSizeTracker; - this.loggerFactory = loggerFactory; - this.logger = loggerFactory?.CreateLogger("StoreWrapper"); - this.sessionLogger = loggerFactory?.CreateLogger("Session"); + this.logger = logger; + this.sessionLogger = logger; // TODO Change map size to a reasonable number this.versionMap = new WatchVersionMap(1 << 16); this.accessControlList = accessControlList; From 7445f16ceb97889a3a7f9350f12a1e49ca104073 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Tue, 21 Jan 2025 06:14:40 +0100 Subject: [PATCH 11/16] fix some unit tests --- libs/host/GarnetApplication.cs | 6 + libs/host/GarnetApplicationBuilder.cs | 115 +++++++++++++++++- .../ClusterAuthCommsTests.cs | 7 +- .../ClusterManagementTests.cs | 7 +- .../ClusterReplicationTests.cs | 38 +++--- .../Garnet.test.cluster/ClusterTestContext.cs | 18 ++- test/Garnet.test/TestUtils.cs | 25 +++- 7 files changed, 179 insertions(+), 37 deletions(-) diff --git a/libs/host/GarnetApplication.cs b/libs/host/GarnetApplication.cs index c95eccc849..436b4dda45 100644 --- a/libs/host/GarnetApplication.cs +++ b/libs/host/GarnetApplication.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Garnet.server; using Microsoft.Extensions.Hosting; namespace Garnet.host; @@ -48,4 +49,9 @@ public static GarnetApplicationBuilder CreateHostBuilder(string[] args) { Args = args }); + + public static GarnetApplicationBuilder CreateHostBuilder(string[] args, GarnetServerOptions options) + { + return new GarnetApplicationBuilder(new GarnetApplicationOptions {Args = args}, options); + } } \ No newline at end of file diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 5b5a09a566..4b49d7b51e 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -28,6 +28,115 @@ public class GarnetApplicationBuilder : IHostApplicationBuilder /// private readonly string redisProtocolVersion = "7.2.5"; + internal GarnetApplicationBuilder(GarnetApplicationOptions options, GarnetServerOptions garnetServerOptions) + { + var configuration = new ConfigurationManager(); + + configuration.AddEnvironmentVariables(prefix: "GARNET_"); + + hostApplicationBuilder = new HostApplicationBuilder(new HostApplicationBuilderSettings + { + Args = options.Args, + ApplicationName = options.ApplicationName, + EnvironmentName = options.EnvironmentName, + Configuration = configuration + }); + + hostApplicationBuilder.Logging.ClearProviders(); + hostApplicationBuilder.Logging.AddSimpleConsole(simpleConsoleFormatterOptions => + { + simpleConsoleFormatterOptions.SingleLine = true; + simpleConsoleFormatterOptions.TimestampFormat = "hh::mm::ss "; + }); + + hostApplicationBuilder.Services.AddOptions(); + + hostApplicationBuilder.Services.AddSingleton(); + + MemoryLogger initLogger; + + // Set up an initial memory logger to log messages from configuration parser into memory. + using (var memLogProvider = new MemoryLoggerProvider()) + { + initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); + } + + if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, + out var exitGracefully, initLogger)) + { + if (exitGracefully) + System.Environment.Exit(0); + + // Flush logs from memory logger + //FlushMemoryLogger(this.initLogger, "ArgParser"); + + throw new GarnetException( + "Encountered an error when initializing Garnet server. Please see log messages above for more details."); + } + + var garnetServerOptionsWrapped = Microsoft.Extensions.Options.Options.Create(garnetServerOptions); + hostApplicationBuilder.Services.AddSingleton(garnetServerOptionsWrapped); + + hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddSingleton(); + + hostApplicationBuilder.Services.AddSingleton(sp => + { + var server = sp.GetRequiredService(); + var opts = sp.GetRequiredService>(); + var customCommandManager = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + var clusterFactory = opts.Value.EnableCluster ? sp.GetRequiredService() : null; + var storeFactory = sp.GetRequiredService(); + + var version = GetVersion(); + + var store = storeFactory.CreateMainStore(out var checkpointDir); + var objectStore = storeFactory.CreateObjectStore(checkpointDir, out var objectStoreSizeTracker); + + TsavoriteLog appendOnlyFile = null; + + if (opts.Value.EnableAOF) + { + if (opts.Value.MainMemoryReplication && opts.Value.CommitFrequencyMs != -1) + throw new Exception( + "Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); + + opts.Value.GetAofSettings(out var aofSettings); + + var tsavoriteLogLogger = sp.GetRequiredService>(); + + appendOnlyFile = new TsavoriteLog(aofSettings, tsavoriteLogLogger); + + if (opts.Value.CommitFrequencyMs < 0 && opts.Value.WaitForCommit) + throw new Exception("Cannot use CommitWait with manual commits"); + } + else + { + if (opts.Value.CommitFrequencyMs != 0 || opts.Value.WaitForCommit) + throw new Exception("Cannot use CommitFrequencyMs or CommitWait without EnableAOF"); + } + + var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); + if (objectStore != null) + configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + + (objectStoreSizeTracker?.TargetSize ?? 0) + + (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); + logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); + + LoadModules(customCommandManager, opts.Value, logger); + + return new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, + customCommandManager, appendOnlyFile, opts.Value, logger, clusterFactory: clusterFactory); + }); + + hostApplicationBuilder.Services.AddHostedService(); + } + internal GarnetApplicationBuilder(GarnetApplicationOptions options) { var configuration = new ConfigurationManager(); @@ -73,7 +182,7 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) throw new GarnetException( "Encountered an error when initializing Garnet server. Please see log messages above for more details."); } - + var garnetServerOptions = serverSettings.GetServerOptions(null); var garnetServerOptionsWrapped = Microsoft.Extensions.Options.Options.Create(garnetServerOptions); hostApplicationBuilder.Services.AddSingleton(garnetServerOptionsWrapped); @@ -107,7 +216,7 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) opts.Value.GetAofSettings(out var aofSettings); var tsavoriteLogLogger = sp.GetRequiredService>(); - + appendOnlyFile = new TsavoriteLog(aofSettings, tsavoriteLogLogger); if (opts.Value.CommitFrequencyMs < 0 && opts.Value.WaitForCommit) @@ -136,7 +245,7 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) }); hostApplicationBuilder.Services.AddHostedService(); - + /* hostApplicationBuilder.Services.AddSingleton(sp => { diff --git a/test/Garnet.test.cluster/ClusterAuthCommsTests.cs b/test/Garnet.test.cluster/ClusterAuthCommsTests.cs index 2f26bcae7f..a3ae372459 100644 --- a/test/Garnet.test.cluster/ClusterAuthCommsTests.cs +++ b/test/Garnet.test.cluster/ClusterAuthCommsTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -195,7 +196,7 @@ public void ClusterSimpleFailoverAuth() [Test, Order(4)] [Category("CLUSTER-AUTH"), CancelAfter(60000)] - public void ClusterSimpleACLReload() + public async Task ClusterSimpleACLReload() { ClusterStartupWithoutAuthCreds(useDefaultUserForInterNodeComms: true); @@ -214,9 +215,9 @@ public void ClusterSimpleACLReload() context.clusterTestUtils.AclLoad(1, logger: context.logger); // Restart node with new ACL file - context.nodes[0].Dispose(false); + await context.nodes[0].StopAsync(); context.nodes[0] = context.CreateInstance(context.clusterTestUtils.GetEndPoint(0).Port, useAcl: true, cleanClusterConfig: false); - context.nodes[0].Start(); + _ = context.nodes[0].RunAsync(); context.CreateConnection(clientCreds: cc[0]); diff --git a/test/Garnet.test.cluster/ClusterManagementTests.cs b/test/Garnet.test.cluster/ClusterManagementTests.cs index dcaff8956b..3bf6587a8c 100644 --- a/test/Garnet.test.cluster/ClusterManagementTests.cs +++ b/test/Garnet.test.cluster/ClusterManagementTests.cs @@ -376,7 +376,7 @@ public void ClusterKeySlotTest() //[Test, Order(5)] //[Category("CLUSTER")] - public void ClusterRestartNodeDropGossip() + public async Task ClusterRestartNodeDropGossip(CancellationToken cancellationToken = default) { var logger = context.loggerFactory.CreateLogger("ClusterRestartNodeDropGossip"); context.CreateInstances(defaultShards); @@ -385,7 +385,7 @@ public void ClusterRestartNodeDropGossip() var restartingNode = 2; // Dispose node and delete data - context.nodes[restartingNode].Dispose(deleteDir: true); + await context.nodes[restartingNode].StopAsync(cancellationToken); context.nodes[restartingNode] = context.CreateInstance( context.clusterTestUtils.GetEndPoint(restartingNode).Port, @@ -395,7 +395,8 @@ public void ClusterRestartNodeDropGossip() timeout: 60, gossipDelay: 1, cleanClusterConfig: false); - context.nodes[restartingNode].Start(); + + _ = context.nodes[restartingNode].RunAsync(cancellationToken); context.CreateConnection(); Thread.Sleep(5000); diff --git a/test/Garnet.test.cluster/ClusterReplicationTests.cs b/test/Garnet.test.cluster/ClusterReplicationTests.cs index cdccac8a3c..f4218173b8 100644 --- a/test/Garnet.test.cluster/ClusterReplicationTests.cs +++ b/test/Garnet.test.cluster/ClusterReplicationTests.cs @@ -142,7 +142,7 @@ public void ClusterSRTest([Values] bool disableObjects) [Test, Order(2)] [Category("REPLICATION")] - public void ClusterSRNoCheckpointRestartSecondary([Values] bool performRMW, [Values] bool disableObjects) + public async Task ClusterSRNoCheckpointRestartSecondary([Values] bool performRMW, [Values] bool disableObjects) { var replica_count = 1;// Per primary var primary_count = 1; @@ -179,7 +179,7 @@ public void ClusterSRNoCheckpointRestartSecondary([Values] bool performRMW, [Val context.ValidateKVCollectionAgainstReplica(ref context.kvPairs, 1); // Shutdown secondary - context.nodes[1].Dispose(false); + await context.nodes[1].StopAsync(); Thread.Sleep(TimeSpan.FromSeconds(2)); @@ -198,7 +198,7 @@ public void ClusterSRNoCheckpointRestartSecondary([Values] bool performRMW, [Val timeout: timeout, useTLS: useTLS, cleanClusterConfig: false); - context.nodes[1].Start(); + _ = context.nodes[1].RunAsync(); context.CreateConnection(useTLS: useTLS); // Validate synchronization was success @@ -208,7 +208,7 @@ public void ClusterSRNoCheckpointRestartSecondary([Values] bool performRMW, [Val [Test, Order(3)] [Category("REPLICATION")] - public void ClusterSRPrimaryCheckpoint([Values] bool performRMW, [Values] bool disableObjects) + public async Task ClusterSRPrimaryCheckpoint([Values] bool performRMW, [Values] bool disableObjects) { var replica_count = 1;// Per primary var primary_count = 1; @@ -253,7 +253,7 @@ public void ClusterSRPrimaryCheckpoint([Values] bool performRMW, [Values] bool d context.clusterTestUtils.WaitCheckpoint(1, replicaLastSaveTime, logger: context.logger); // Shutdown secondary - context.nodes[1].Dispose(false); + await context.nodes[1].StopAsync(); Thread.Sleep(TimeSpan.FromSeconds(2)); // New insert @@ -272,7 +272,7 @@ public void ClusterSRPrimaryCheckpoint([Values] bool performRMW, [Values] bool d useTLS: useTLS, cleanClusterConfig: false, asyncReplay: asyncReplay); - context.nodes[1].Start(); + _ = context.nodes[1].RunAsync(); context.CreateConnection(useTLS: useTLS); for (int i = 1; i < replica_count; i++) context.clusterTestUtils.WaitForReplicaRecovery(i, context.logger); @@ -302,7 +302,7 @@ public void ClusterCheckpointRetrieveDelta([Values] bool performRMW) public void ClusterSRPrimaryCheckpointRetrieve([Values] bool performRMW, [Values] bool disableObjects, [Values] bool lowMemory, [Values] bool manySegments) => ClusterSRPrimaryCheckpointRetrieve(performRMW: performRMW, disableObjects: disableObjects, lowMemory: lowMemory, manySegments: manySegments, false, false); - void ClusterSRPrimaryCheckpointRetrieve(bool performRMW, bool disableObjects, bool lowMemory, bool manySegments, bool disableStorageTier, bool incrementalSnapshots) + async Task ClusterSRPrimaryCheckpointRetrieve(bool performRMW, bool disableObjects, bool lowMemory, bool manySegments, bool disableStorageTier, bool incrementalSnapshots) { // Test many segments on or off with lowMemory manySegments = lowMemory && manySegments; @@ -335,7 +335,7 @@ void ClusterSRPrimaryCheckpointRetrieve(bool performRMW, bool disableObjects, bo context.kvPairsObj = []; context.logger?.LogTrace("Test disposing node 1"); - context.nodes[1].Dispose(false); + await context.nodes[1].StopAsync(); Thread.Sleep(TimeSpan.FromSeconds(1)); // Populate Primary @@ -371,7 +371,7 @@ void ClusterSRPrimaryCheckpointRetrieve(bool performRMW, bool disableObjects, bo SegmentSize: manySegments ? "4k" : "1g", DisableStorageTier: disableStorageTier, asyncReplay: asyncReplay); - context.nodes[replicaIndex].Start(); + _ = context.nodes[replicaIndex].RunAsync(); context.CreateConnection(useTLS: useTLS); context.clusterTestUtils.WaitForReplicaAofSync(primaryIndex, replicaIndex, context.logger); @@ -442,7 +442,7 @@ public void ClusterSRAddReplicaAfterPrimaryCheckpoint([Values] bool performRMW, [Test, Order(8)] [Category("REPLICATION")] - public void ClusterSRPrimaryRestart([Values] bool performRMW, [Values] bool disableObjects) + public async Task ClusterSRPrimaryRestart([Values] bool performRMW, [Values] bool disableObjects) { var replica_count = 1;// Per primary var primary_count = 1; @@ -481,7 +481,7 @@ public void ClusterSRPrimaryRestart([Values] bool performRMW, [Values] bool disa if (!disableObjects) objectStoreCurrentAofAddress = context.clusterTestUtils.GetObjectStoreCurrentAofAddress(0, context.logger); - context.nodes[0].Dispose(false); + await context.nodes[0].StopAsync(); Thread.Sleep(TimeSpan.FromSeconds(1)); // Restart Primary @@ -494,7 +494,7 @@ public void ClusterSRPrimaryRestart([Values] bool performRMW, [Values] bool disa useTLS: useTLS, cleanClusterConfig: false, asyncReplay: asyncReplay); - context.nodes[0].Start(); + _ = context.nodes[0].RunAsync(); context.CreateConnection(useTLS: useTLS); var storeRecoveredAofAddress = context.clusterTestUtils.GetStoreRecoveredAofAddress(0, context.logger); @@ -901,7 +901,7 @@ public void ClusterDivergentCheckpointMMFastCommitTest([Values] bool disableObje mainMemoryReplication: mainMemoryReplication, fastCommit: true); - void ClusterDivergentReplicasTest(bool performRMW, bool disableObjects, bool ckptBeforeDivergence, bool multiCheckpointAfterDivergence, bool mainMemoryReplication, bool fastCommit) + async Task ClusterDivergentReplicasTest(bool performRMW, bool disableObjects, bool ckptBeforeDivergence, bool multiCheckpointAfterDivergence, bool mainMemoryReplication, bool fastCommit) { var set = false; var replica_count = 2;// Per primary @@ -979,7 +979,7 @@ void ClusterDivergentReplicasTest(bool performRMW, bool disableObjects, bool ckp context.clusterTestUtils.WaitForReplicaAofSync(oldPrimaryIndex, replicaIndex, context.logger); // Dispose primary - context.nodes[oldPrimaryIndex].Dispose(false); + await context.nodes[oldPrimaryIndex].StopAsync(); context.nodes[oldPrimaryIndex] = null; // Re-assign slots to replica manually since failover option was not @@ -1063,7 +1063,7 @@ public void ClusterReplicateFails() } [Test, Order(22)] - public void ClusterReplicationCheckpointAlignmentTest([Values] bool performRMW) + public async Task ClusterReplicationCheckpointAlignmentTest([Values] bool performRMW) { var replica_count = 1;// Per primary var primary_count = 1; @@ -1105,9 +1105,9 @@ public void ClusterReplicationCheckpointAlignmentTest([Values] bool performRMW) context.ValidateKVCollectionAgainstReplica(ref context.kvPairs, replicaNodeIndex); // Dispose primary and delete data - context.nodes[primaryNodeIndex].Dispose(true); + await context.nodes[primaryNodeIndex].StopAsync(); // Dispose primary but do not delete data - context.nodes[replicaNodeIndex].Dispose(false); + await context.nodes[replicaNodeIndex].StopAsync(); // Restart primary and do not recover context.nodes[primaryNodeIndex] = context.CreateInstance( @@ -1122,7 +1122,7 @@ public void ClusterReplicationCheckpointAlignmentTest([Values] bool performRMW) useTLS: useTLS, cleanClusterConfig: true, asyncReplay: asyncReplay); - context.nodes[primaryNodeIndex].Start(); + context.nodes[primaryNodeIndex].Run(); // Restart secondary and recover context.nodes[replicaNodeIndex] = context.CreateInstance( @@ -1137,7 +1137,7 @@ public void ClusterReplicationCheckpointAlignmentTest([Values] bool performRMW) useTLS: useTLS, cleanClusterConfig: true, asyncReplay: asyncReplay); - context.nodes[replicaNodeIndex].Start(); + context.nodes[replicaNodeIndex].Run(); context.CreateConnection(useTLS: useTLS); // Assert primary version is 1 and replica has recovered to previous checkpoint diff --git a/test/Garnet.test.cluster/ClusterTestContext.cs b/test/Garnet.test.cluster/ClusterTestContext.cs index 8b1529879e..9022e65f59 100644 --- a/test/Garnet.test.cluster/ClusterTestContext.cs +++ b/test/Garnet.test.cluster/ClusterTestContext.cs @@ -5,13 +5,17 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Reflection; using System.Security.Cryptography.X509Certificates; using System.Text; using System.Threading; using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using Garnet.server.Auth.Settings; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -22,7 +26,7 @@ public class ClusterTestContext { public CredentialManager credManager; public string TestFolder; - public GarnetServer[] nodes = null; + public GarnetApplication[] nodes = null; public EndPointCollection endpoints; public TextWriter logTextWriter = TestContext.Progress; public ILoggerFactory loggerFactory; @@ -72,8 +76,10 @@ public void TearDown() public void RegisterCustomTxn(string name, Func proc, RespCommandsInfo commandInfo = null, RespCommandDocs commandDocs = null) { + /* foreach (var node in nodes) node.Register.NewTransactionProc(name, proc, commandInfo, commandDocs); + */ } /// @@ -162,7 +168,7 @@ public void CreateInstances( asyncReplay: asyncReplay); foreach (var node in nodes) - node.Start(); + _ = node.RunAsync(); } /// @@ -191,7 +197,7 @@ public void CreateInstances( /// /// /// - public GarnetServer CreateInstance( + public GarnetApplication CreateInstance( int Port, bool cleanClusterConfig = true, bool disableEpochCollision = false, @@ -248,7 +254,11 @@ public GarnetServer CreateInstance( authPassword: clusterCreds.password, certificates: certificates); - return new GarnetServer(opts, loggerFactory); + var builder = GarnetApplication.CreateHostBuilder([], opts); + + var app = builder.Build(); + + return app; } diff --git a/test/Garnet.test/TestUtils.cs b/test/Garnet.test/TestUtils.cs index 221843ce97..7e9698a494 100644 --- a/test/Garnet.test/TestUtils.cs +++ b/test/Garnet.test/TestUtils.cs @@ -14,12 +14,14 @@ using System.Threading; using Garnet.client; using Garnet.common; +using Garnet.host; using Garnet.server; using Garnet.server.Auth.Settings; using Garnet.server.TLS; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.Text; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -336,11 +338,19 @@ public static GarnetServer CreateGarnetServer( builder.SetMinimumLevel(LogLevel.Trace); }); - return new GarnetServer(opts, loggerFactory); + var builder = GarnetApplication.CreateHostBuilder([], opts); + + var app = builder.Build(); + + return app.Services.GetRequiredService(); } else { - return new GarnetServer(opts); + var builder = GarnetApplication.CreateHostBuilder([], opts); + + var app = builder.Build(); + + return app.Services.GetRequiredService(); } } @@ -364,7 +374,7 @@ public static ILoggerFactory CreateLoggerFactoryInstance(TextWriter textWriter, }); } - public static GarnetServer[] CreateGarnetCluster( + public static GarnetApplication[] CreateGarnetCluster( string checkpointDir, EndPointCollection endpoints, bool disablePubSub = false, @@ -400,7 +410,7 @@ public static GarnetServer[] CreateGarnetCluster( { if (UseAzureStorage) IgnoreIfNotRunningAzureTests(); - GarnetServer[] nodes = new GarnetServer[endpoints.Count]; + var nodes = new GarnetApplication[endpoints.Count]; for (int i = 0; i < nodes.Length; i++) { IPEndPoint endpoint = (IPEndPoint)endpoints[i]; @@ -448,7 +458,12 @@ public static GarnetServer[] CreateGarnetCluster( TestContext.Progress.WriteLine($"Waiting for Port {opts.Port} to become available for {TestContext.CurrentContext.WorkerId}:{iter++}"); Thread.Sleep(1000); } - nodes[i] = new GarnetServer(opts, loggerFactory); + + var builder = GarnetApplication.CreateHostBuilder([], opts); + + var app = builder.Build(); + + nodes[i] = app; } return nodes; } From 6bfe05b3736bfa2cb7334a3072b7d384b14afb00 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Tue, 21 Jan 2025 06:26:36 +0100 Subject: [PATCH 12/16] refactor CreateHostBuilder methods --- libs/host/GarnetApplication.cs | 35 +++++-- libs/host/GarnetApplicationBuilder.cs | 143 -------------------------- 2 files changed, 29 insertions(+), 149 deletions(-) diff --git a/libs/host/GarnetApplication.cs b/libs/host/GarnetApplication.cs index 436b4dda45..ece31f9b0e 100644 --- a/libs/host/GarnetApplication.cs +++ b/libs/host/GarnetApplication.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Garnet.common; using Garnet.server; using Microsoft.Extensions.Hosting; @@ -40,15 +41,37 @@ public async Task RunAsync(CancellationToken cancellationToken = default) { await HostingAbstractionsHostExtensions.RunAsync(this, cancellationToken); } - + public static GarnetApplicationBuilder CreateHostBuilder() - => new(new()); - + => CreateHostBuilder([]); + public static GarnetApplicationBuilder CreateHostBuilder(string[] args) - => new(new() + { + MemoryLogger initLogger; + + // Set up an initial memory logger to log messages from configuration parser into memory. + using (var memLogProvider = new MemoryLoggerProvider()) + { + initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); + } + + if (!ServerSettingsManager.TryParseCommandLineArguments(args, out var serverSettings, out _, + out var exitGracefully, initLogger)) { - Args = args - }); + if (exitGracefully) + System.Environment.Exit(0); + + // Flush logs from memory logger + //FlushMemoryLogger(this.initLogger, "ArgParser"); + + throw new GarnetException( + "Encountered an error when initializing Garnet server. Please see log messages above for more details."); + } + + var garnetServerOptions = serverSettings.GetServerOptions(null); + + return new GarnetApplicationBuilder(new GarnetApplicationOptions {Args = args}, garnetServerOptions); + } public static GarnetApplicationBuilder CreateHostBuilder(string[] args, GarnetServerOptions options) { diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 4b49d7b51e..b3c019a97b 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -53,137 +53,6 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options, GarnetServer hostApplicationBuilder.Services.AddSingleton(); - MemoryLogger initLogger; - - // Set up an initial memory logger to log messages from configuration parser into memory. - using (var memLogProvider = new MemoryLoggerProvider()) - { - initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); - } - - if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, - out var exitGracefully, initLogger)) - { - if (exitGracefully) - System.Environment.Exit(0); - - // Flush logs from memory logger - //FlushMemoryLogger(this.initLogger, "ArgParser"); - - throw new GarnetException( - "Encountered an error when initializing Garnet server. Please see log messages above for more details."); - } - - var garnetServerOptionsWrapped = Microsoft.Extensions.Options.Options.Create(garnetServerOptions); - hostApplicationBuilder.Services.AddSingleton(garnetServerOptionsWrapped); - - hostApplicationBuilder.Services.AddSingleton(); - hostApplicationBuilder.Services.AddSingleton(); - hostApplicationBuilder.Services.AddSingleton(); - - hostApplicationBuilder.Services.AddSingleton(sp => - { - var server = sp.GetRequiredService(); - var opts = sp.GetRequiredService>(); - var customCommandManager = sp.GetRequiredService(); - var logger = sp.GetRequiredService>(); - var clusterFactory = opts.Value.EnableCluster ? sp.GetRequiredService() : null; - var storeFactory = sp.GetRequiredService(); - - var version = GetVersion(); - - var store = storeFactory.CreateMainStore(out var checkpointDir); - var objectStore = storeFactory.CreateObjectStore(checkpointDir, out var objectStoreSizeTracker); - - TsavoriteLog appendOnlyFile = null; - - if (opts.Value.EnableAOF) - { - if (opts.Value.MainMemoryReplication && opts.Value.CommitFrequencyMs != -1) - throw new Exception( - "Need to set CommitFrequencyMs to -1 (manual commits) with MainMemoryReplication"); - - opts.Value.GetAofSettings(out var aofSettings); - - var tsavoriteLogLogger = sp.GetRequiredService>(); - - appendOnlyFile = new TsavoriteLog(aofSettings, tsavoriteLogLogger); - - if (opts.Value.CommitFrequencyMs < 0 && opts.Value.WaitForCommit) - throw new Exception("Cannot use CommitWait with manual commits"); - } - else - { - if (opts.Value.CommitFrequencyMs != 0 || opts.Value.WaitForCommit) - throw new Exception("Cannot use CommitFrequencyMs or CommitWait without EnableAOF"); - } - - var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + - (store.ReadCache?.MaxMemorySizeBytes ?? 0) + - (appendOnlyFile?.MaxMemorySizeBytes ?? 0); - if (objectStore != null) - configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + - (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + - (objectStoreSizeTracker?.TargetSize ?? 0) + - (objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); - logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); - - LoadModules(customCommandManager, opts.Value, logger); - - return new StoreWrapper(version, redisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, - customCommandManager, appendOnlyFile, opts.Value, logger, clusterFactory: clusterFactory); - }); - - hostApplicationBuilder.Services.AddHostedService(); - } - - internal GarnetApplicationBuilder(GarnetApplicationOptions options) - { - var configuration = new ConfigurationManager(); - - configuration.AddEnvironmentVariables(prefix: "GARNET_"); - - hostApplicationBuilder = new HostApplicationBuilder(new HostApplicationBuilderSettings - { - Args = options.Args, - ApplicationName = options.ApplicationName, - EnvironmentName = options.EnvironmentName, - Configuration = configuration - }); - - hostApplicationBuilder.Logging.ClearProviders(); - hostApplicationBuilder.Logging.AddSimpleConsole(simpleConsoleFormatterOptions => - { - simpleConsoleFormatterOptions.SingleLine = true; - simpleConsoleFormatterOptions.TimestampFormat = "hh::mm::ss "; - }); - - hostApplicationBuilder.Services.AddOptions(); - - hostApplicationBuilder.Services.AddSingleton(); - - MemoryLogger initLogger; - - // Set up an initial memory logger to log messages from configuration parser into memory. - using (var memLogProvider = new MemoryLoggerProvider()) - { - initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); - } - - if (!ServerSettingsManager.TryParseCommandLineArguments(options.Args, out var serverSettings, out _, - out var exitGracefully, initLogger)) - { - if (exitGracefully) - System.Environment.Exit(0); - - // Flush logs from memory logger - //FlushMemoryLogger(this.initLogger, "ArgParser"); - - throw new GarnetException( - "Encountered an error when initializing Garnet server. Please see log messages above for more details."); - } - - var garnetServerOptions = serverSettings.GetServerOptions(null); var garnetServerOptionsWrapped = Microsoft.Extensions.Options.Options.Create(garnetServerOptions); hostApplicationBuilder.Services.AddSingleton(garnetServerOptionsWrapped); @@ -245,18 +114,6 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options) }); hostApplicationBuilder.Services.AddHostedService(); - - /* - hostApplicationBuilder.Services.AddSingleton(sp => - { - var opts = sp.GetRequiredService>(); - var wrapper = sp.GetRequiredService(); - var server = sp.GetRequiredService(); - var logger = sp.GetRequiredService>(); - - return new GarnetServerMonitor(wrapper, opts.Value, server, logger); - }); - */ } public GarnetApplication Build() From 8bb82a9ebb5a0d853af4bf4f2f6b03dd70a2f455 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Tue, 21 Jan 2025 06:45:23 +0100 Subject: [PATCH 13/16] add registration --- libs/host/GarnetApplication.cs | 7 ++----- main/GarnetServer/Program.cs | 7 +++++++ test/Garnet.test.cluster/ClusterTestContext.cs | 10 +++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/libs/host/GarnetApplication.cs b/libs/host/GarnetApplication.cs index ece31f9b0e..ba2482bd3f 100644 --- a/libs/host/GarnetApplication.cs +++ b/libs/host/GarnetApplication.cs @@ -42,9 +42,6 @@ public async Task RunAsync(CancellationToken cancellationToken = default) await HostingAbstractionsHostExtensions.RunAsync(this, cancellationToken); } - public static GarnetApplicationBuilder CreateHostBuilder() - => CreateHostBuilder([]); - public static GarnetApplicationBuilder CreateHostBuilder(string[] args) { MemoryLogger initLogger; @@ -70,11 +67,11 @@ public static GarnetApplicationBuilder CreateHostBuilder(string[] args) var garnetServerOptions = serverSettings.GetServerOptions(null); - return new GarnetApplicationBuilder(new GarnetApplicationOptions {Args = args}, garnetServerOptions); + return new (new GarnetApplicationOptions {Args = args}, garnetServerOptions); } public static GarnetApplicationBuilder CreateHostBuilder(string[] args, GarnetServerOptions options) { - return new GarnetApplicationBuilder(new GarnetApplicationOptions {Args = args}, options); + return new (new GarnetApplicationOptions {Args = args}, options); } } \ No newline at end of file diff --git a/main/GarnetServer/Program.cs b/main/GarnetServer/Program.cs index b7e71ce22c..e232e67857 100644 --- a/main/GarnetServer/Program.cs +++ b/main/GarnetServer/Program.cs @@ -1,14 +1,21 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System.Linq; using Garnet; using Garnet.host; using Garnet.server; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; var builder = GarnetApplication.CreateHostBuilder(args); var app = builder.Build(); +var hostedServices = app.Services.GetServices(); +var server = hostedServices.OfType().FirstOrDefault(); +RegisterExtensions(server); + app.Run(); /// diff --git a/test/Garnet.test.cluster/ClusterTestContext.cs b/test/Garnet.test.cluster/ClusterTestContext.cs index 9022e65f59..f14466a145 100644 --- a/test/Garnet.test.cluster/ClusterTestContext.cs +++ b/test/Garnet.test.cluster/ClusterTestContext.cs @@ -14,6 +14,7 @@ using Garnet.server; using Garnet.server.Auth.Settings; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NUnit.Framework; @@ -76,10 +77,13 @@ public void TearDown() public void RegisterCustomTxn(string name, Func proc, RespCommandsInfo commandInfo = null, RespCommandDocs commandDocs = null) { - /* foreach (var node in nodes) - node.Register.NewTransactionProc(name, proc, commandInfo, commandDocs); - */ + { + var hostedServices = node.Services.GetServices(); + var server = hostedServices.OfType().FirstOrDefault(); + + server.Register.NewTransactionProc(name, proc, commandInfo, commandDocs); + } } /// From d7c37bf854f49e43224f546494fa0a3e4ff41301 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Tue, 21 Jan 2025 06:56:50 +0100 Subject: [PATCH 14/16] refactor --- libs/host/GarnetApplicationBuilder.cs | 16 +++++++++++ libs/host/GarnetServer.cs | 40 +++++---------------------- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index b3c019a97b..118b43ade4 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -30,6 +30,22 @@ public class GarnetApplicationBuilder : IHostApplicationBuilder internal GarnetApplicationBuilder(GarnetApplicationOptions options, GarnetServerOptions garnetServerOptions) { + var version = GetVersion(); + + if (!garnetServerOptions.QuietMode) + { + var red = "\u001b[31m"; + var magenta = "\u001b[35m"; + var normal = "\u001b[0m"; + + Console.WriteLine($@"{red} _________ + /_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(garnetServerOptions.EnableCluster ? "cluster" : "standalone")} mode{red} + '. \ / .' {normal}Port: {garnetServerOptions.Port}{red} + '.\ /.' {magenta}https://aka.ms/GetGarnet{red} + '.' + {normal}"); + } + var configuration = new ConfigurationManager(); configuration.AddEnvironmentVariables(prefix: "GARNET_"); diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 662d511bbd..5fb2f20944 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -21,13 +21,6 @@ namespace Garnet; /// public class GarnetServer : IHostedService, IDisposable { - static readonly string version = GetVersion(); - static string GetVersion() - { - var Version = Assembly.GetExecutingAssembly().GetName().Version; - return $"{Version.Major}.{Version.Minor}.{Version.Build}"; - } - internal GarnetProvider Provider; private readonly GarnetServerOptions opts; @@ -74,17 +67,6 @@ public GarnetServer( this.initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); } - if (!ServerSettingsManager.TryParseCommandLineArguments([], out var serverSettings, out _, out var exitGracefully, this.initLogger)) - { - if (exitGracefully) - Environment.Exit(0); - - // Flush logs from memory logger - FlushMemoryLogger(this.initLogger, "ArgParser"); - - throw new GarnetException("Encountered an error when initializing Garnet server. Please see log messages above for more details."); - } - // Assign values to GarnetServerOptions this.opts = opts.Value; this.opts.AuthSettings = this.opts.AuthSettings; @@ -94,24 +76,10 @@ public GarnetServer( private void InitializeServer() { Debug.Assert(opts != null); - - if (!opts.QuietMode) - { - var red = "\u001b[31m"; - var magenta = "\u001b[35m"; - var normal = "\u001b[0m"; - - Console.WriteLine($@"{red} _________ - /_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(opts.EnableCluster ? "cluster" : "standalone")} mode{red} - '. \ / .' {normal}Port: {opts.Port}{red} - '.\ /.' {magenta}https://aka.ms/GetGarnet{red} - '.' - {normal}"); - } using (logger.BeginScope("GarnetServer")) { - logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", version, IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.Port); + logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", GetVersion(), IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.Port); // Flush initialization logs from memory logger FlushMemoryLogger(this.initLogger, "ArgParser"); @@ -217,4 +185,10 @@ public Task StopAsync(CancellationToken cancellationToken) return Task.CompletedTask; } + + private static string GetVersion() + { + var Version = Assembly.GetExecutingAssembly().GetName().Version; + return $"{Version.Major}.{Version.Minor}.{Version.Build}"; + } } \ No newline at end of file From 4b55f59a83410b22318ba9b7224dc23542d71ab2 Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Wed, 22 Jan 2025 01:47:58 +0100 Subject: [PATCH 15/16] move more of the initialization code out of GarnetServer constructor --- .../Embedded/EmbeddedRespServer.cs | 12 +++- .../BDN.benchmark/Lua/LuaRunnerOperations.cs | 9 +++ hosting/Windows/Garnet.worker/Program.cs | 22 +++---- libs/host/GarnetApplication.cs | 61 +++++++++++++++++- libs/host/GarnetApplicationBuilder.cs | 42 ++++++------ libs/host/GarnetServer.cs | 64 +++---------------- libs/server/PubSub/SubscribeBroker.cs | 8 +++ 7 files changed, 127 insertions(+), 91 deletions(-) diff --git a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs index 186e442f9a..d0989eb3bc 100644 --- a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs +++ b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs @@ -4,6 +4,7 @@ using Garnet; using Garnet.server; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Embedded.server { @@ -18,10 +19,15 @@ internal sealed class EmbeddedRespServer : GarnetServer /// Creates an EmbeddedRespServer instance /// /// Server options to configure the base GarnetServer instance - /// Logger factory to configure the base GarnetServer instance + /// Logger factory to configure the base GarnetServer instance /// Server network - public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, GarnetServerEmbedded server = null) - : base(opts, loggerFactory, server) + /// Server network + public EmbeddedRespServer( + IOptions opts, + ILogger logger, + GarnetServerEmbedded server, + StoreWrapper store) + : base(opts, logger, server, store) { this.garnetServerEmbedded = server; } diff --git a/benchmark/BDN.benchmark/Lua/LuaRunnerOperations.cs b/benchmark/BDN.benchmark/Lua/LuaRunnerOperations.cs index 07d28e641b..4cb2c7f2a3 100644 --- a/benchmark/BDN.benchmark/Lua/LuaRunnerOperations.cs +++ b/benchmark/BDN.benchmark/Lua/LuaRunnerOperations.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Attributes; using Embedded.server; +using Garnet.host; using Garnet.server; namespace BDN.benchmark.Lua @@ -154,6 +155,14 @@ public IEnumerable LuaParamsProvider() [GlobalSetup] public void GlobalSetup() { + var builder = GarnetApplication.CreateHostBuilder([], new GarnetServerOptions() + { + EnableLua = true, + QuietMode = true + }); + + var app = builder.Build(); + server = new EmbeddedRespServer(new GarnetServerOptions() { EnableLua = true, QuietMode = true }); session = server.GetRespSession(); diff --git a/hosting/Windows/Garnet.worker/Program.cs b/hosting/Windows/Garnet.worker/Program.cs index 8418da8671..a4253e7b79 100644 --- a/hosting/Windows/Garnet.worker/Program.cs +++ b/hosting/Windows/Garnet.worker/Program.cs @@ -5,19 +5,13 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -class Program -{ - static void Main(string[] args) - { - var builder = Host.CreateApplicationBuilder(args); - builder.Services.AddHostedService(_ => new Worker(args)); +var builder = Host.CreateApplicationBuilder(args); +builder.Services.AddHostedService(_ => new Worker(args)); - builder.Services.AddWindowsService(options => - { - options.ServiceName = "Microsoft Garnet Server"; - }); +builder.Services.AddWindowsService(options => +{ + options.ServiceName = "Microsoft Garnet Server"; +}); - var host = builder.Build(); - host.Run(); - } -} \ No newline at end of file +var host = builder.Build(); +host.Run(); \ No newline at end of file diff --git a/libs/host/GarnetApplication.cs b/libs/host/GarnetApplication.cs index ba2482bd3f..58efc453c7 100644 --- a/libs/host/GarnetApplication.cs +++ b/libs/host/GarnetApplication.cs @@ -2,11 +2,16 @@ // Licensed under the MIT license. using System; +using System.Diagnostics; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Garnet.common; using Garnet.server; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Garnet.host; @@ -25,7 +30,53 @@ public GarnetApplication(IHost host) public IServiceProvider Services => host.Services; public Task StartAsync(CancellationToken cancellationToken = default) - => host.StartAsync(cancellationToken); + { + var opts = host.Services.GetRequiredService>(); + var logger = host.Services.GetRequiredService>(); + + Debug.Assert(opts != null); + + var version = GetVersion(); + + if (!opts.Value.QuietMode) + { + var red = "\u001b[31m"; + var magenta = "\u001b[35m"; + var normal = "\u001b[0m"; + + Console.WriteLine($@"{red} _________ + /_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(opts.Value.EnableCluster ? "cluster" : "standalone")} mode{red} + '. \ / .' {normal}Port: {opts.Value.Port}{red} + '.\ /.' {magenta}https://aka.ms/GetGarnet{red} + '.' + {normal}"); + } + + Trace.Listeners.Add(new ConsoleTraceListener()); + + // Set up an initial memory logger to log messages from configuration parser into memory. + using var memLogProvider = new MemoryLoggerProvider(); + + var initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); + + logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", GetVersion(), IntPtr.Size == 8 ? "64" : "32", opts.Value.EnableCluster ? "cluster" : "standalone", opts.Value.Port); + + // Flush initialization logs from memory logger + initLogger.FlushLogger(logger); + + var setMax = opts.Value.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.Value.ThreadPoolMaxThreads, opts.Value.ThreadPoolMaxThreads); + + if (opts.Value.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.Value.ThreadPoolMinThreads, opts.Value.ThreadPoolMinThreads)) + throw new Exception($"Unable to call ThreadPool.SetMinThreads with {opts.Value.ThreadPoolMinThreads}"); + + // Retry to set max threads if it wasn't set in the previous step + if (!setMax && !ThreadPool.SetMaxThreads(opts.Value.ThreadPoolMaxThreads, opts.Value.ThreadPoolMaxThreads)) + throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.Value.ThreadPoolMaxThreads}"); + + logger?.LogTrace("TLS is {tlsEnabled}", opts.Value.TlsOptions == null ? "disabled" : "enabled"); + + return host.StartAsync(cancellationToken); + } public Task StopAsync(CancellationToken cancellationToken = default) => host.StopAsync(cancellationToken); @@ -59,7 +110,7 @@ public static GarnetApplicationBuilder CreateHostBuilder(string[] args) System.Environment.Exit(0); // Flush logs from memory logger - //FlushMemoryLogger(this.initLogger, "ArgParser"); + //initLogger.FlushLogger(logger); throw new GarnetException( "Encountered an error when initializing Garnet server. Please see log messages above for more details."); @@ -74,4 +125,10 @@ public static GarnetApplicationBuilder CreateHostBuilder(string[] args, GarnetSe { return new (new GarnetApplicationOptions {Args = args}, options); } + + private static string GetVersion() + { + var Version = Assembly.GetExecutingAssembly().GetName().Version; + return $"{Version.Major}.{Version.Minor}.{Version.Build}"; + } } \ No newline at end of file diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 118b43ade4..3f7c0a6819 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -30,22 +30,6 @@ public class GarnetApplicationBuilder : IHostApplicationBuilder internal GarnetApplicationBuilder(GarnetApplicationOptions options, GarnetServerOptions garnetServerOptions) { - var version = GetVersion(); - - if (!garnetServerOptions.QuietMode) - { - var red = "\u001b[31m"; - var magenta = "\u001b[35m"; - var normal = "\u001b[0m"; - - Console.WriteLine($@"{red} _________ - /_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(garnetServerOptions.EnableCluster ? "cluster" : "standalone")} mode{red} - '. \ / .' {normal}Port: {garnetServerOptions.Port}{red} - '.\ /.' {magenta}https://aka.ms/GetGarnet{red} - '.' - {normal}"); - } - var configuration = new ConfigurationManager(); configuration.AddEnvironmentVariables(prefix: "GARNET_"); @@ -72,13 +56,13 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options, GarnetServer var garnetServerOptionsWrapped = Microsoft.Extensions.Options.Options.Create(garnetServerOptions); hostApplicationBuilder.Services.AddSingleton(garnetServerOptionsWrapped); - hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddSingleton(); hostApplicationBuilder.Services.AddSingleton(); hostApplicationBuilder.Services.AddSingleton(); hostApplicationBuilder.Services.AddSingleton(sp => { - var server = sp.GetRequiredService(); + var server = sp.GetRequiredService(); var opts = sp.GetRequiredService>(); var customCommandManager = sp.GetRequiredService(); var logger = sp.GetRequiredService>(); @@ -129,6 +113,28 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options, GarnetServer customCommandManager, appendOnlyFile, opts.Value, logger, clusterFactory: clusterFactory); }); + if (!garnetServerOptions.DisablePubSub) + { + hostApplicationBuilder.Services + .AddSingleton(sp => + { + var opts = sp.GetRequiredService>(); + return SubscribeBroker>.Create(opts); + }); + } + + hostApplicationBuilder.Services.AddSingleton(sp => + { + var storeWrapper = sp.GetRequiredService(); + var subscriberBroker = sp.GetService>>(); + + return new GarnetProvider(storeWrapper, subscriberBroker); + }); + + hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddHostedService(); } diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 5fb2f20944..c471691e51 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -26,7 +26,6 @@ public class GarnetServer : IHostedService, IDisposable private readonly GarnetServerOptions opts; private IGarnetServer server; private SubscribeBroker> subscribeBroker; - private readonly MemoryLogger initLogger; private readonly ILogger logger; /// @@ -52,7 +51,7 @@ public class GarnetServer : IHostedService, IDisposable public GarnetServer( IOptions opts, ILogger logger, - GarnetServerTcp garnetServerTcp, + IGarnetServer garnetServerTcp, StoreWrapper storeWrapper) { this.logger = logger; @@ -61,53 +60,24 @@ public GarnetServer( Trace.Listeners.Add(new ConsoleTraceListener()); - // Set up an initial memory logger to log messages from configuration parser into memory. - using (var memLogProvider = new MemoryLoggerProvider()) - { - this.initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); - } - // Assign values to GarnetServerOptions this.opts = opts.Value; this.opts.AuthSettings = this.opts.AuthSettings; + this.InitializeServer(); } private void InitializeServer() { - Debug.Assert(opts != null); - - using (logger.BeginScope("GarnetServer")) - { - logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", GetVersion(), IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.Port); - - // Flush initialization logs from memory logger - FlushMemoryLogger(this.initLogger, "ArgParser"); - - var setMax = opts.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads); - - if (opts.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.ThreadPoolMinThreads, opts.ThreadPoolMinThreads)) - throw new Exception($"Unable to call ThreadPool.SetMinThreads with {opts.ThreadPoolMinThreads}"); - - // Retry to set max threads if it wasn't set in the previous step - if (!setMax && !ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads)) - throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.ThreadPoolMaxThreads}"); - - if (!opts.DisablePubSub) - subscribeBroker = new SubscribeBroker>(new SpanByteKeySerializer(), null, opts.PubSubPageSizeBytes(), opts.SubscriberRefreshFrequencyMs, true); + // Create session provider for Garnet + Provider = new GarnetProvider(storeWrapper, subscribeBroker); - logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled"); + // Create user facing API endpoints + Metrics = new MetricsApi(Provider); + Register = new RegisterApi(Provider); + Store = new StoreApi(storeWrapper); - // Create session provider for Garnet - Provider = new GarnetProvider(storeWrapper, subscribeBroker); - - // Create user facing API endpoints - Metrics = new MetricsApi(Provider); - Register = new RegisterApi(Provider); - Store = new StoreApi(storeWrapper); - - server.Register(WireFormat.ASCII, Provider); - } + server.Register(WireFormat.ASCII, Provider); } /// @@ -119,7 +89,7 @@ public void Start() server.Start(); Provider.Start(); if (!opts.QuietMode) - Console.WriteLine("* Ready to accept connections"); + this.logger.LogInformation("* Ready to accept connections"); } /// @@ -158,20 +128,6 @@ private void InternalDispose() opts.AuthSettings?.Dispose(); } - /// - /// Flushes MemoryLogger entries into a destination logger. - /// Destination logger is either created from ILoggerFactory parameter or from a default console logger. - /// - /// The memory logger - /// The category name of the destination logger - private void FlushMemoryLogger(MemoryLogger memoryLogger, string categoryName) - { - using (this.logger.BeginScope(categoryName)) - { - memoryLogger.FlushLogger(this.logger); - } - } - public Task StartAsync(CancellationToken cancellationToken) { Start(); diff --git a/libs/server/PubSub/SubscribeBroker.cs b/libs/server/PubSub/SubscribeBroker.cs index 53d073b922..915ef76eca 100644 --- a/libs/server/PubSub/SubscribeBroker.cs +++ b/libs/server/PubSub/SubscribeBroker.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Garnet.common; using Garnet.networking; +using Microsoft.Extensions.Options; using Tsavorite.core; namespace Garnet.server @@ -35,6 +36,13 @@ public sealed class SubscribeBroker : IDispos readonly ManualResetEvent done = new(true); bool disposed = false; + public static SubscribeBroker> Create( + IOptions options) + { + return new SubscribeBroker>(new SpanByteKeySerializer(), null, + options.Value.PubSubPageSizeBytes(), options.Value.SubscriberRefreshFrequencyMs, true); + } + /// /// Constructor /// From 92395181c38741a03f162d5d30a834e235bd4d2d Mon Sep 17 00:00:00 2001 From: Tomas Pelak Date: Wed, 22 Jan 2025 03:36:07 +0100 Subject: [PATCH 16/16] fix tests --- hosting/Windows/Garnet.worker/Worker.cs | 2 +- libs/host/GarnetApplication.cs | 92 ++++------ libs/host/GarnetApplicationBuilder.cs | 6 +- libs/host/GarnetServer.cs | 169 +++++++----------- libs/server/StoreWrapper.cs | 2 +- main/GarnetServer/Program.cs | 9 +- .../Garnet.test.cluster/ClusterTestContext.cs | 5 +- test/Garnet.test/CacheSizeTrackerTests.cs | 18 +- test/Garnet.test/GarnetBitmapTests.cs | 36 ++-- test/Garnet.test/GarnetClientTests.cs | 22 +-- .../GarnetJSON/JsonCommandsTest.cs | 28 +-- test/Garnet.test/HyperLogLogTests.cs | 30 ++-- test/Garnet.test/IndexGrowthTests.cs | 28 +-- test/Garnet.test/LuaScriptTests.cs | 7 +- test/Garnet.test/ObjectTestsForOutput.cs | 7 +- test/Garnet.test/ReadCacheTests.cs | 8 +- .../Resp/ACL/AclConfigurationFileTests.cs | 16 +- test/Garnet.test/Resp/ACL/AclTest.cs | 3 +- test/Garnet.test/Resp/ACL/BasicTests.cs | 4 +- test/Garnet.test/Resp/ACL/DeleteUserTests.cs | 4 +- test/Garnet.test/Resp/ACL/ParallelTests.cs | 4 +- test/Garnet.test/Resp/ACL/RespCommandTests.cs | 11 +- test/Garnet.test/Resp/ACL/SetUserTests.cs | 36 ++-- .../Resp/GarnetAuthenticatorTests.cs | 5 +- test/Garnet.test/RespAdminCommandsTests.cs | 67 +++---- test/Garnet.test/RespAofAzureTests.cs | 79 ++++---- test/Garnet.test/RespAofTests.cs | 105 +++++------ test/Garnet.test/RespBlockingListTests.cs | 11 +- test/Garnet.test/RespCommandTests.cs | 12 +- test/Garnet.test/RespCustomCommandTests.cs | 11 +- test/Garnet.test/RespGetLowMemoryTests.cs | 11 +- test/Garnet.test/RespHashTests.cs | 11 +- test/Garnet.test/RespInfoTests.cs | 12 +- test/Garnet.test/RespListGarnetClientTests.cs | 11 +- test/Garnet.test/RespListTests.cs | 11 +- test/Garnet.test/RespLowMemoryTests.cs | 12 +- test/Garnet.test/RespMetricsTest.cs | 12 +- test/Garnet.test/RespModuleTests.cs | 16 +- test/Garnet.test/RespPubSubTests.cs | 12 +- test/Garnet.test/RespScanCommandsTests.cs | 12 +- test/Garnet.test/RespSetTest.cs | 11 +- .../RespSortedSetGarnetClientTests.cs | 11 +- test/Garnet.test/RespSortedSetGeoTests.cs | 12 +- test/Garnet.test/RespSortedSetTests.cs | 11 +- test/Garnet.test/RespTests.cs | 39 ++-- test/Garnet.test/RespTlsTests.cs | 11 +- test/Garnet.test/RespTransactionProcTests.cs | 12 +- test/Garnet.test/TestUtils.cs | 6 +- test/Garnet.test/TransactionTests.cs | 15 +- 49 files changed, 531 insertions(+), 554 deletions(-) diff --git a/hosting/Windows/Garnet.worker/Worker.cs b/hosting/Windows/Garnet.worker/Worker.cs index b6de5ea2ec..e4e6dd5997 100644 --- a/hosting/Windows/Garnet.worker/Worker.cs +++ b/hosting/Windows/Garnet.worker/Worker.cs @@ -14,7 +14,7 @@ public class Worker : BackgroundService private bool _isDisposed = false; private readonly string[] args; - private GarnetServer server; + private GarnetApplication server; public Worker(string[] args) { diff --git a/libs/host/GarnetApplication.cs b/libs/host/GarnetApplication.cs index 58efc453c7..a9ea80bf12 100644 --- a/libs/host/GarnetApplication.cs +++ b/libs/host/GarnetApplication.cs @@ -2,15 +2,12 @@ // Licensed under the MIT license. using System; -using System.Diagnostics; -using System.Reflection; using System.Threading; using System.Threading.Tasks; using Garnet.common; using Garnet.server; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Garnet.host; @@ -20,77 +17,58 @@ namespace Garnet.host; /// public class GarnetApplication : IHost { + /// + /// Metrics API + /// + public MetricsApi Metrics; + + /// + /// Command registration API + /// + public RegisterApi Register; + + /// + /// Store API + /// + public StoreApi Store; + + internal GarnetProvider Provider; + private readonly IHost host; public GarnetApplication(IHost host) { this.host = host; + + Metrics = host.Services.GetRequiredService(); + Register = host.Services.GetService(); + Store = host.Services.GetService(); + + Provider = host.Services.GetRequiredService(); } public IServiceProvider Services => host.Services; public Task StartAsync(CancellationToken cancellationToken = default) - { - var opts = host.Services.GetRequiredService>(); - var logger = host.Services.GetRequiredService>(); - - Debug.Assert(opts != null); - - var version = GetVersion(); - - if (!opts.Value.QuietMode) - { - var red = "\u001b[31m"; - var magenta = "\u001b[35m"; - var normal = "\u001b[0m"; - - Console.WriteLine($@"{red} _________ - /_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(opts.Value.EnableCluster ? "cluster" : "standalone")} mode{red} - '. \ / .' {normal}Port: {opts.Value.Port}{red} - '.\ /.' {magenta}https://aka.ms/GetGarnet{red} - '.' - {normal}"); - } - - Trace.Listeners.Add(new ConsoleTraceListener()); - - // Set up an initial memory logger to log messages from configuration parser into memory. - using var memLogProvider = new MemoryLoggerProvider(); - - var initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); - - logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", GetVersion(), IntPtr.Size == 8 ? "64" : "32", opts.Value.EnableCluster ? "cluster" : "standalone", opts.Value.Port); - - // Flush initialization logs from memory logger - initLogger.FlushLogger(logger); - - var setMax = opts.Value.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.Value.ThreadPoolMaxThreads, opts.Value.ThreadPoolMaxThreads); - - if (opts.Value.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.Value.ThreadPoolMinThreads, opts.Value.ThreadPoolMinThreads)) - throw new Exception($"Unable to call ThreadPool.SetMinThreads with {opts.Value.ThreadPoolMinThreads}"); - - // Retry to set max threads if it wasn't set in the previous step - if (!setMax && !ThreadPool.SetMaxThreads(opts.Value.ThreadPoolMaxThreads, opts.Value.ThreadPoolMaxThreads)) - throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.Value.ThreadPoolMaxThreads}"); - - logger?.LogTrace("TLS is {tlsEnabled}", opts.Value.TlsOptions == null ? "disabled" : "enabled"); - - return host.StartAsync(cancellationToken); - } + => host.StartAsync(cancellationToken); public Task StopAsync(CancellationToken cancellationToken = default) => host.StopAsync(cancellationToken); - - public void Dispose() => host.Dispose(); + + public void Dispose() + { + host.Dispose(); + } public void Run() { HostingAbstractionsHostExtensions.Run(this); } - public async Task RunAsync(CancellationToken cancellationToken = default) + public Task RunAsync(CancellationToken cancellationToken = default) { - await HostingAbstractionsHostExtensions.RunAsync(this, cancellationToken); + HostingAbstractionsHostExtensions.RunAsync(this, cancellationToken); + return Task.CompletedTask; } public static GarnetApplicationBuilder CreateHostBuilder(string[] args) @@ -125,10 +103,4 @@ public static GarnetApplicationBuilder CreateHostBuilder(string[] args, GarnetSe { return new (new GarnetApplicationOptions {Args = args}, options); } - - private static string GetVersion() - { - var Version = Assembly.GetExecutingAssembly().GetName().Version; - return $"{Version.Major}.{Version.Minor}.{Version.Build}"; - } } \ No newline at end of file diff --git a/libs/host/GarnetApplicationBuilder.cs b/libs/host/GarnetApplicationBuilder.cs index 3f7c0a6819..45c0f3e941 100644 --- a/libs/host/GarnetApplicationBuilder.cs +++ b/libs/host/GarnetApplicationBuilder.cs @@ -57,6 +57,8 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options, GarnetServer hostApplicationBuilder.Services.AddSingleton(garnetServerOptionsWrapped); hostApplicationBuilder.Services.AddSingleton(); + hostApplicationBuilder.Services.AddHostedService(); + hostApplicationBuilder.Services.AddSingleton(); hostApplicationBuilder.Services.AddSingleton(); @@ -73,7 +75,7 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options, GarnetServer var store = storeFactory.CreateMainStore(out var checkpointDir); var objectStore = storeFactory.CreateObjectStore(checkpointDir, out var objectStoreSizeTracker); - + TsavoriteLog appendOnlyFile = null; if (opts.Value.EnableAOF) @@ -134,8 +136,6 @@ internal GarnetApplicationBuilder(GarnetApplicationOptions options, GarnetServer hostApplicationBuilder.Services.AddSingleton(); hostApplicationBuilder.Services.AddSingleton(); hostApplicationBuilder.Services.AddSingleton(); - - hostApplicationBuilder.Services.AddHostedService(); } public GarnetApplication Build() diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index c471691e51..d21b152759 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -6,140 +6,97 @@ using System.Reflection; using System.Threading; using System.Threading.Tasks; -using Garnet.common; using Garnet.networking; using Garnet.server; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Tsavorite.core; -namespace Garnet; +namespace Garnet.host; -/// -/// Implementation Garnet server -/// -public class GarnetServer : IHostedService, IDisposable +public class GarnetServer : IHostedService { - internal GarnetProvider Provider; - - private readonly GarnetServerOptions opts; - private IGarnetServer server; - private SubscribeBroker> subscribeBroker; + private readonly IGarnetServer garnetServerTcp; + private readonly GarnetProvider garnetProvider; + private readonly StoreWrapper store; + private readonly IOptions opts; private readonly ILogger logger; - - /// - /// Store and associated information used by this Garnet server - /// - protected StoreWrapper storeWrapper; - - /// - /// Metrics API - /// - public MetricsApi Metrics; - - /// - /// Command registration API - /// - public RegisterApi Register; - - /// - /// Store API - /// - public StoreApi Store; - + public GarnetServer( - IOptions opts, - ILogger logger, - IGarnetServer garnetServerTcp, - StoreWrapper storeWrapper) + IGarnetServer garnetServerTcp, + GarnetProvider garnetProvider, + StoreWrapper store, + IOptions options, + ILogger logger) { + this.garnetServerTcp = garnetServerTcp; + this.garnetProvider = garnetProvider; + this.store = store; + this.opts = options; this.logger = logger; - this.server = garnetServerTcp; - this.storeWrapper = storeWrapper; - Trace.Listeners.Add(new ConsoleTraceListener()); - - // Assign values to GarnetServerOptions - this.opts = opts.Value; - this.opts.AuthSettings = this.opts.AuthSettings; + garnetServerTcp.Register(WireFormat.ASCII, garnetProvider); - this.InitializeServer(); } - private void InitializeServer() - { - // Create session provider for Garnet - Provider = new GarnetProvider(storeWrapper, subscribeBroker); + public Task StartAsync(CancellationToken cancellationToken) + { + var version = GetVersion(); + + if (!opts.Value.QuietMode) + { + var red = "\u001b[31m"; + var magenta = "\u001b[35m"; + var normal = "\u001b[0m"; + + Console.WriteLine($@"{red} _________ + /_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(opts.Value.EnableCluster ? "cluster" : "standalone")} mode{red} + '. \ / .' {normal}Port: {opts.Value.Port}{red} + '.\ /.' {magenta}https://aka.ms/GetGarnet{red} + '.' + {normal}"); + } + + Trace.Listeners.Add(new ConsoleTraceListener()); - // Create user facing API endpoints - Metrics = new MetricsApi(Provider); - Register = new RegisterApi(Provider); - Store = new StoreApi(storeWrapper); + // Set up an initial memory logger to log messages from configuration parser into memory. + using var memLogProvider = new MemoryLoggerProvider(); + + var initLogger = (MemoryLogger)memLogProvider.CreateLogger("ArgParser"); + + logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Port: {port}", GetVersion(), IntPtr.Size == 8 ? "64" : "32", opts.Value.EnableCluster ? "cluster" : "standalone", opts.Value.Port); - server.Register(WireFormat.ASCII, Provider); - } + // Flush initialization logs from memory logger + initLogger.FlushLogger(logger); - /// - /// Start server instance - /// - public void Start() - { - Provider.Recover(); - server.Start(); - Provider.Start(); - if (!opts.QuietMode) - this.logger.LogInformation("* Ready to accept connections"); - } + var setMax = opts.Value.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.Value.ThreadPoolMaxThreads, opts.Value.ThreadPoolMaxThreads); - /// - /// Dispose store (including log and checkpoint directory) - /// - public void Dispose() - { - Dispose(false); - } + if (opts.Value.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.Value.ThreadPoolMinThreads, opts.Value.ThreadPoolMinThreads)) + throw new Exception($"Unable to call ThreadPool.SetMinThreads with {opts.Value.ThreadPoolMinThreads}"); - /// - /// Dispose, optionally deleting logs and checkpoints - /// - /// Whether to delete logs and checkpoints - public void Dispose(bool deleteDir = true) - { - InternalDispose(); - if (deleteDir) - { - if (opts.CheckpointDir != opts.LogDir && !string.IsNullOrEmpty(opts.CheckpointDir)) - { - var ckptdir = opts.DeviceFactoryCreator(); - ckptdir.Initialize(opts.CheckpointDir); - ckptdir.Delete(new FileDescriptor { directoryName = "" }); - } - } - } + // Retry to set max threads if it wasn't set in the previous step + if (!setMax && !ThreadPool.SetMaxThreads(opts.Value.ThreadPoolMaxThreads, opts.Value.ThreadPoolMaxThreads)) + throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.Value.ThreadPoolMaxThreads}"); - private void InternalDispose() - { - // Provider.Dispose will get stuck - //Provider?.Dispose(); + logger?.LogTrace("TLS is {tlsEnabled}", opts.Value.TlsOptions == null ? "disabled" : "enabled"); + + garnetProvider.Recover(); + garnetServerTcp.Start(); + garnetProvider.Start(); + + if (!opts.Value.QuietMode) + logger?.LogInformation("* Ready to accept connections"); - server.Dispose(); - subscribeBroker?.Dispose(); - opts.AuthSettings?.Dispose(); - } - - public Task StartAsync(CancellationToken cancellationToken) - { - Start(); - return Task.CompletedTask; } public Task StopAsync(CancellationToken cancellationToken) { - Dispose(); + garnetProvider?.Dispose(); + garnetServerTcp?.Dispose(); + //store?.Dispose(); - return Task.CompletedTask; + return Task.CompletedTask; } private static string GetVersion() diff --git a/libs/server/StoreWrapper.cs b/libs/server/StoreWrapper.cs index c6856f92f9..724770bbae 100644 --- a/libs/server/StoreWrapper.cs +++ b/libs/server/StoreWrapper.cs @@ -26,7 +26,7 @@ namespace Garnet.server /// /// Wrapper for store and store-specific information /// - public sealed class StoreWrapper + public sealed class StoreWrapper { internal readonly string version; internal readonly string redisProtocolVersion; diff --git a/main/GarnetServer/Program.cs b/main/GarnetServer/Program.cs index e232e67857..8ded59d120 100644 --- a/main/GarnetServer/Program.cs +++ b/main/GarnetServer/Program.cs @@ -1,20 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Linq; using Garnet; using Garnet.host; using Garnet.server; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; var builder = GarnetApplication.CreateHostBuilder(args); var app = builder.Build(); -var hostedServices = app.Services.GetServices(); -var server = hostedServices.OfType().FirstOrDefault(); -RegisterExtensions(server); +RegisterExtensions(app); app.Run(); @@ -23,7 +18,7 @@ /// commands such as db.Execute in StackExchange.Redis. Example: /// db.Execute("SETIFPM", key, value, prefix); /// -static void RegisterExtensions(GarnetServer server) +static void RegisterExtensions(GarnetApplication server) { // Register custom command on raw strings (SETIFPM = "set if prefix match") // Add RESP command info to registration for command to appear when client runs COMMAND / COMMAND INFO diff --git a/test/Garnet.test.cluster/ClusterTestContext.cs b/test/Garnet.test.cluster/ClusterTestContext.cs index f14466a145..36a774a47b 100644 --- a/test/Garnet.test.cluster/ClusterTestContext.cs +++ b/test/Garnet.test.cluster/ClusterTestContext.cs @@ -79,10 +79,7 @@ public void RegisterCustomTxn(string name, Func proc { foreach (var node in nodes) { - var hostedServices = node.Services.GetServices(); - var server = hostedServices.OfType().FirstOrDefault(); - - server.Register.NewTransactionProc(name, proc, commandInfo, commandDocs); + node.Register.NewTransactionProc(name, proc, commandInfo, commandDocs); } } diff --git a/test/Garnet.test/CacheSizeTrackerTests.cs b/test/Garnet.test/CacheSizeTrackerTests.cs index 41dfabddf5..020c139064 100644 --- a/test/Garnet.test/CacheSizeTrackerTests.cs +++ b/test/Garnet.test/CacheSizeTrackerTests.cs @@ -3,6 +3,8 @@ using System; using System.Threading; +using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -17,24 +19,24 @@ namespace Garnet.test [TestFixture] public class CacheSizeTrackerTests { - GarnetServer server; + GarnetApplication server; TsavoriteKV objStore; CacheSizeTracker cacheSizeTracker; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, MemorySize: "2k", PageSize: "512", lowMemory: true, objectStoreIndexSize: "1k", objectStoreHeapMemorySize: "5k"); - server.Start(); + await server.RunAsync(); objStore = server.Provider.StoreWrapper.objectStore; cacheSizeTracker = server.Provider.StoreWrapper.objectStoreSizeTracker; } [TearDown] - public void TearDown() + public async Task TearDown() { - server?.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } @@ -90,11 +92,11 @@ public void IncreaseEmptyPageCountTest() } [Test] - public void ReadCacheIncreaseEmptyPageCountTest() + public async Task ReadCacheIncreaseEmptyPageCountTest() { - server?.Dispose(); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, MemorySize: "1k", PageSize: "512", lowMemory: true, objectStoreIndexSize: "1k", objectStoreReadCacheHeapMemorySize: "1k", enableObjectStoreReadCache: true); - server.Start(); + await server.RunAsync(); objStore = server.Provider.StoreWrapper.objectStore; cacheSizeTracker = server.Provider.StoreWrapper.objectStoreSizeTracker; diff --git a/test/Garnet.test/GarnetBitmapTests.cs b/test/Garnet.test/GarnetBitmapTests.cs index c1281a26d4..87f7a94921 100644 --- a/test/Garnet.test/GarnetBitmapTests.cs +++ b/test/Garnet.test/GarnetBitmapTests.cs @@ -3,7 +3,9 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Garnet.common; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -13,22 +15,22 @@ namespace Garnet.test { public class GarnetBitmapTests { - GarnetServer server; + GarnetApplication server; Random r; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); r = new Random(674386); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } @@ -162,7 +164,7 @@ public void BitmapSimpleSetGet_PCT(int bytesPerSend) [TestCase(false)] [TestCase(true)] [Category("SET+GET+BIT")] - public void BitmapSetGetBitTest_LTM(bool preSet) + public async Task BitmapSetGetBitTest_LTM(bool preSet) { int bitmapBytes = 512; server.Dispose(); @@ -170,7 +172,7 @@ public void BitmapSetGetBitTest_LTM(bool preSet) lowMemory: true, MemorySize: (bitmapBytes << 2).ToString(), PageSize: (bitmapBytes << 1).ToString()); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -444,7 +446,7 @@ public void BitmapBitCountNegativeOffsets() [Test, Order(10)] [Category("BITCOUNT")] - public void BitmapBitCountTest_LTM() + public async Task BitmapBitCountTest_LTM() { int bitmapBytes = 512; server.Dispose(); @@ -452,7 +454,7 @@ public void BitmapBitCountTest_LTM() lowMemory: true, MemorySize: (bitmapBytes << 2).ToString(), PageSize: (bitmapBytes << 1).ToString()); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -641,7 +643,7 @@ public void BitmapBitPosOffsetsTest() [Test, Order(14)] [Category("BITPOS")] - public void BitmapBitPosTest_LTM() + public async Task BitmapBitPosTest_LTM() { int bitmapBytes = 512; server.Dispose(); @@ -649,7 +651,7 @@ public void BitmapBitPosTest_LTM() lowMemory: true, MemorySize: (bitmapBytes << 2).ToString(), PageSize: (bitmapBytes << 1).ToString()); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -1264,7 +1266,7 @@ public unsafe void BitmapBitfieldGetTest_PCT([Values(RespCommand.BITFIELD, RespC [Test, Order(23)] [Category("BITFIELD")] - public void BitmapBitfieldGetTest_LTM([Values(RespCommand.BITFIELD, RespCommand.BITFIELD_RO)] RespCommand testCmd) + public async Task BitmapBitfieldGetTest_LTM([Values(RespCommand.BITFIELD, RespCommand.BITFIELD_RO)] RespCommand testCmd) { int bitmapBytes = 512; server.Dispose(); @@ -1274,7 +1276,7 @@ public void BitmapBitfieldGetTest_LTM([Values(RespCommand.BITFIELD, RespCommand. PageSize: (bitmapBytes << 1).ToString()); //MemorySize: "16g", //PageSize: "32m"); - server.Start(); + await server.StopAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -1465,7 +1467,7 @@ public void BitmapBitfieldSetTest() [Test, Order(26)] [Category("BITFIELD")] - public void BitmapBitfieldSetTest_LTM() + public async Task BitmapBitfieldSetTest_LTM() { int bitmapBytes = 512; server.Dispose(); @@ -1475,7 +1477,7 @@ public void BitmapBitfieldSetTest_LTM() PageSize: (bitmapBytes << 1).ToString()); //MemorySize: "16g", //PageSize: "32m"); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -1931,7 +1933,7 @@ public void BitmapBitfieldSignedIncrTest() [Test, Order(29)] [Category("BITFIELD")] - public void BitmapBitfieldIncrTest_LTM() + public async Task BitmapBitfieldIncrTest_LTM() { int bitmapBytes = 512; server.Dispose(); @@ -1941,7 +1943,7 @@ public void BitmapBitfieldIncrTest_LTM() PageSize: (bitmapBytes << 1).ToString()); //MemorySize: "16g", //PageSize: "32m"); - server.Start(); + await server.StopAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); diff --git a/test/Garnet.test/GarnetClientTests.cs b/test/Garnet.test/GarnetClientTests.cs index 56498b2fcc..c129853839 100644 --- a/test/Garnet.test/GarnetClientTests.cs +++ b/test/Garnet.test/GarnetClientTests.cs @@ -117,10 +117,10 @@ static void WaitAndReset(ManualResetEventSlim e) } [Test] - public void SetGetWithCallback([Values] bool useTLS) + public async Task SetGetWithCallback([Values] bool useTLS) { using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, EnableTLS: useTLS); - server.Start(); + await server.RunAsync(); using var db = TestUtils.GetGarnetClient(useTLS); db.Connect(); @@ -145,10 +145,10 @@ public void SetGetWithCallback([Values] bool useTLS) } [Test] - public void SimpleMetricsTest([Values] bool recordLatency) + public async Task SimpleMetricsTest([Values] bool recordLatency) { using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); var db = TestUtils.GetGarnetClient(recordLatency: recordLatency); db.Connect(); @@ -180,7 +180,7 @@ public void SimpleMetricsTest([Values] bool recordLatency) public async Task SimpleStringArrayTest([Values] bool stringParams) { using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); var db = TestUtils.GetGarnetClient(); db.Connect(); @@ -196,7 +196,7 @@ public async Task SimpleStringArrayTest([Values] bool stringParams) public async Task SimpleNoArgsTest() { using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); var db = TestUtils.GetGarnetClient(); db.Connect(); @@ -213,7 +213,7 @@ public async Task SimpleIncrTest([Values] bool stringParams) { ManualResetEventSlim e = new(); using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); var key = "mykey"; var db = TestUtils.GetGarnetClient(); @@ -275,7 +275,7 @@ public async Task SimpleDecrTest([Values] bool stringParams) { ManualResetEventSlim e = new(); using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); var key = "mykey"; var db = TestUtils.GetGarnetClient(); @@ -336,7 +336,7 @@ public async Task SimpleDecrTest([Values] bool stringParams) public async Task CanUseSetNxStringResultAsync() { using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); var db = TestUtils.GetGarnetClient(); db.Connect(); @@ -355,7 +355,7 @@ public async Task CanUseSetNxStringResultAsync() public async Task CanUseMGetTests([Values] bool disableObjectStore) { using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: disableObjectStore); - server.Start(); + await server.RunAsync(); var db = TestUtils.GetGarnetClient(); db.Connect(); @@ -462,7 +462,7 @@ public async Task CanDoBulkDeleteTests([Values] bool useStringType) { //KeyDeleteAsync using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); var db = TestUtils.GetGarnetClient(); db.Connect(); diff --git a/test/Garnet.test/GarnetJSON/JsonCommandsTest.cs b/test/Garnet.test/GarnetJSON/JsonCommandsTest.cs index 7ed66c9d36..243934c140 100644 --- a/test/Garnet.test/GarnetJSON/JsonCommandsTest.cs +++ b/test/Garnet.test/GarnetJSON/JsonCommandsTest.cs @@ -5,6 +5,8 @@ using System.IO; using System.Reflection; using System.Threading; +using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using GarnetJSON; using NUnit.Framework; @@ -16,22 +18,22 @@ namespace Garnet.test [TestFixture] class JsonCommandsTest { - GarnetServer server; + GarnetApplication server; string binPath; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); binPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, extensionAllowUnsignedAssemblies: true, extensionBinPaths: [binPath]); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } @@ -84,7 +86,7 @@ public void JsonSetGetTests() } [Test] - public void SaveRecoverTest() + public async Task SaveRecoverTest() { string key = "key"; RegisterCustomCommand(); @@ -102,10 +104,10 @@ public void SaveRecoverTest() while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true); RegisterCustomCommand(); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -116,12 +118,12 @@ public void SaveRecoverTest() } [Test] - public void AofUpsertRecoverTest() + public async Task AofUpsertRecoverTest() { - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableAOF: true); RegisterCustomCommand(); - server.Start(); + await server.RunAsync(); var key = "aofkey"; using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) @@ -133,10 +135,10 @@ public void AofUpsertRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); RegisterCustomCommand(); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { diff --git a/test/Garnet.test/HyperLogLogTests.cs b/test/Garnet.test/HyperLogLogTests.cs index 646cfd86d4..1d88ef859b 100644 --- a/test/Garnet.test/HyperLogLogTests.cs +++ b/test/Garnet.test/HyperLogLogTests.cs @@ -5,6 +5,8 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,17 +14,17 @@ namespace Garnet.test { - public unsafe class HyperLogLogTests + public class HyperLogLogTests { - GarnetServer server; + GarnetApplication server; Random r; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); r = new Random(674386); } @@ -329,7 +331,7 @@ private static unsafe ulong MurmurHash2x64A(byte* bString, int len, uint seed = [Test] [Repeat(1)] - public void HyperLogLogUpdateReturnTest() + public unsafe void HyperLogLogUpdateReturnTest() { using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -387,7 +389,7 @@ private void RandomString(ref byte[] valuebuffer) [Test] [Repeat(1)] - public void HyperLogLogMultiValueUpdateReturnTest() + public unsafe void HyperLogLogMultiValueUpdateReturnTest() { using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -573,7 +575,7 @@ public void HyperLogLogTestPFADDV2() [TestCase(32)] [TestCase(4096)] [Repeat(1)] - public void HyperLogLogPFADD_LTM(int seqSize) + public async Task HyperLogLogPFADD_LTM(int seqSize) { bool sparse = seqSize < 128 ? true : false; server.Dispose(); @@ -587,7 +589,7 @@ public void HyperLogLogPFADD_LTM(int seqSize) lowMemory: true, MemorySize: "32k", PageSize: "16k"); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -693,14 +695,14 @@ public void HyperLogLogTestPFMERGE_SparseToSparseV2() [Test] [Repeat(10)] - public void HyperLogLogTestPFMERGE_LTM_SparseToSparse() + public async Task HyperLogLogTestPFMERGE_LTM_SparseToSparse() { server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, MemorySize: "1024", PageSize: "512"); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -803,13 +805,13 @@ public void HyperLogLogTestPFMERGE_SparseToDenseV2() [TestCase(false)] [TestCase(true)] [Repeat(1)] - public void HyperLogLogTestPFMERGE_LTM_SparseToDense(bool reverse) + public async Task HyperLogLogTestPFMERGE_LTM_SparseToDense(bool reverse) { server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, MemorySize: "32k", PageSize: "16k"); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -912,14 +914,14 @@ public void HyperLogLogTestPFMERGE_DenseToDenseV2() [Test] [Repeat(1)] - public void HyperLogLogTestPFMERGE_LTM_DenseToDense() + public async Task HyperLogLogTestPFMERGE_LTM_DenseToDense() { server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, MemorySize: "32k", PageSize: "16k"); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); diff --git a/test/Garnet.test/IndexGrowthTests.cs b/test/Garnet.test/IndexGrowthTests.cs index c08d2f90cb..25e13fc8f4 100644 --- a/test/Garnet.test/IndexGrowthTests.cs +++ b/test/Garnet.test/IndexGrowthTests.cs @@ -3,6 +3,8 @@ using System; using System.Threading; +using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -12,7 +14,7 @@ namespace Garnet.test [TestFixture] public class IndexGrowthTests { - GarnetServer server; + GarnetApplication server; private int indexResizeTaskDelaySeconds = 10; private int indexResizeWaitCycles = 2; @@ -30,10 +32,10 @@ public void TearDown() } [Test] - public void IndexGrowthTest() + public async Task IndexGrowthTest() { server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, indexSize: "64", indexMaxSize: "128", indexResizeFrequencySecs: indexResizeTaskDelaySeconds); - server.Start(); + await server.RunAsync(); var store = server.Provider.StoreWrapper.store; @@ -68,10 +70,10 @@ public void IndexGrowthTest() } [Test] - public void ObjectStoreIndexGrowthTest() + public async Task ObjectStoreIndexGrowthTest() { server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, objectStoreIndexSize: "64", objectStoreIndexMaxSize: "128", indexResizeFrequencySecs: indexResizeTaskDelaySeconds); - server.Start(); + await server.RunAsync(); var objectStore = server.Provider.StoreWrapper.objectStore; @@ -115,10 +117,10 @@ private static void VerifyObjectStoreSetMembers(IDatabase db, RedisKey[] keys, R } [Test] - public void IndexGrowthTestWithDiskReadAndCheckpoint() + public async Task IndexGrowthTestWithDiskReadAndCheckpoint() { server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, indexSize: "512", indexMaxSize: "1k", indexResizeFrequencySecs: indexResizeTaskDelaySeconds); - server.Start(); + await server.RunAsync(); var store = server.Provider.StoreWrapper.store; @@ -166,9 +168,9 @@ public void IndexGrowthTestWithDiskReadAndCheckpoint() while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, lowMemory: true, indexSize: "512", indexMaxSize: "1k"); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -179,10 +181,10 @@ public void IndexGrowthTestWithDiskReadAndCheckpoint() } [Test] - public void ObjectStoreIndexGrowthTestWithDiskReadAndCheckpoint() + public async Task ObjectStoreIndexGrowthTestWithDiskReadAndCheckpoint() { server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, objectStoreIndexSize: "512", objectStoreIndexMaxSize: "1k", indexResizeFrequencySecs: indexResizeTaskDelaySeconds); - server.Start(); + await server.RunAsync(); var objectStore = server.Provider.StoreWrapper.objectStore; @@ -230,9 +232,9 @@ public void ObjectStoreIndexGrowthTestWithDiskReadAndCheckpoint() while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, lowMemory: true, objectStoreIndexSize: "512", objectStoreIndexMaxSize: "1k"); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { diff --git a/test/Garnet.test/LuaScriptTests.cs b/test/Garnet.test/LuaScriptTests.cs index ace0593340..bf018be0db 100644 --- a/test/Garnet.test/LuaScriptTests.cs +++ b/test/Garnet.test/LuaScriptTests.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -14,14 +15,14 @@ namespace Garnet.test [TestFixture] public class LuaScriptTests { - protected GarnetServer server; + protected GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableLua: true); - server.Start(); + await server.RunAsync(); } [TearDown] diff --git a/test/Garnet.test/ObjectTestsForOutput.cs b/test/Garnet.test/ObjectTestsForOutput.cs index bcaf7b256a..e245ac65af 100644 --- a/test/Garnet.test/ObjectTestsForOutput.cs +++ b/test/Garnet.test/ObjectTestsForOutput.cs @@ -3,6 +3,7 @@ using System.Text; using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -11,14 +12,14 @@ namespace Garnet.test [TestFixture] public class ObjectTestsForOutput { - protected GarnetServer server; + protected GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: false); - server.Start(); + await server.RunAsync(); } diff --git a/test/Garnet.test/ReadCacheTests.cs b/test/Garnet.test/ReadCacheTests.cs index 80aa968484..795540fa9b 100644 --- a/test/Garnet.test/ReadCacheTests.cs +++ b/test/Garnet.test/ReadCacheTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -10,14 +12,14 @@ namespace Garnet.test [TestFixture] public class ReadCacheTests { - GarnetServer server; + GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableReadCache: true, enableObjectStoreReadCache: true, lowMemory: true); - server.Start(); + await server.RunAsync(); } [TearDown] diff --git a/test/Garnet.test/Resp/ACL/AclConfigurationFileTests.cs b/test/Garnet.test/Resp/ACL/AclConfigurationFileTests.cs index bac1958332..0e5d0b103e 100644 --- a/test/Garnet.test/Resp/ACL/AclConfigurationFileTests.cs +++ b/test/Garnet.test/Resp/ACL/AclConfigurationFileTests.cs @@ -30,7 +30,7 @@ public async Task EmptyInput() // Ensure Garnet starts up with default user only server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, aclFile: configurationFile); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -52,7 +52,7 @@ public async Task NoDefaultRule() // Start up Garnet server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, aclFile: configurationFile); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -78,7 +78,7 @@ public async Task WithDefaultRule() // Start up Garnet with a defined default user password server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, aclFile: configurationFile, defaultPassword: DummyPassword); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -118,7 +118,7 @@ public async Task AclLoad() // Start up Garnet server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, aclFile: configurationFile, defaultPassword: DummyPassword); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -189,7 +189,7 @@ public async Task AclLoadErrors() // Start up Garnet server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, aclFile: configurationFile); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -244,7 +244,7 @@ public async Task DuplicateUserNames() File.WriteAllText(configurationFile, $"user test on >{DummyPassword} +@admin\r\nuser test off"); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, aclFile: configurationFile); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -328,7 +328,7 @@ public void BadInputMalformedStatement() } [Test] - public void AclSave() + public async Task AclSave() { // Create a modified ACL that (1) removes two users, (2) adds one user, (3) removes one password and (4) removes the default user var originalConfigurationFile = @@ -343,7 +343,7 @@ public void AclSave() // Start up Garnet server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, aclFile: configurationFile, defaultPassword: DummyPassword); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); diff --git a/test/Garnet.test/Resp/ACL/AclTest.cs b/test/Garnet.test/Resp/ACL/AclTest.cs index 9f5e614cda..1063c76920 100644 --- a/test/Garnet.test/Resp/ACL/AclTest.cs +++ b/test/Garnet.test/Resp/ACL/AclTest.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System.IO; +using Garnet.host; using NUnit.Framework; namespace Garnet.test.Resp.ACL @@ -44,7 +45,7 @@ abstract class AclTest /// /// Garnet server instance to use in the tests. /// - protected GarnetServer server = null; + protected GarnetApplication server = null; /// /// Creates working directory diff --git a/test/Garnet.test/Resp/ACL/BasicTests.cs b/test/Garnet.test/Resp/ACL/BasicTests.cs index bdbaa63651..9593a140a1 100644 --- a/test/Garnet.test/Resp/ACL/BasicTests.cs +++ b/test/Garnet.test/Resp/ACL/BasicTests.cs @@ -19,10 +19,10 @@ internal class BasicTests : AclTest /// Creates and starts the Garnet test server /// [SetUp] - public virtual void Setup() + public virtual async Task Setup() { server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); } /// diff --git a/test/Garnet.test/Resp/ACL/DeleteUserTests.cs b/test/Garnet.test/Resp/ACL/DeleteUserTests.cs index 69b142d32c..aeebdd6815 100644 --- a/test/Garnet.test/Resp/ACL/DeleteUserTests.cs +++ b/test/Garnet.test/Resp/ACL/DeleteUserTests.cs @@ -18,10 +18,10 @@ class DeleteUserTests : AclTest /// Creates and starts the Garnet test server /// [SetUp] - public virtual void Setup() + public virtual async Task Setup() { server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); } /// diff --git a/test/Garnet.test/Resp/ACL/ParallelTests.cs b/test/Garnet.test/Resp/ACL/ParallelTests.cs index 129ff99027..d7a8354fe3 100644 --- a/test/Garnet.test/Resp/ACL/ParallelTests.cs +++ b/test/Garnet.test/Resp/ACL/ParallelTests.cs @@ -18,10 +18,10 @@ internal class ParallelTests : AclTest /// Creates and starts the Garnet test server /// [SetUp] - public virtual void Setup() + public virtual async Task Setup() { server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); } /// diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index bf47124bdb..f8ea8c7178 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -8,6 +8,7 @@ using System.Reflection; using System.Threading.Tasks; using Garnet.client; +using Garnet.host; using Garnet.server; using Garnet.server.ACL; using NUnit.Framework; @@ -22,11 +23,11 @@ public class RespCommandTests private IReadOnlyDictionary respCustomCommandsInfo; - private GarnetServer server; + private GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, defaultPassword: DefaultPassword, useAcl: true, enableLua: true); @@ -40,13 +41,13 @@ public void Setup() server.Register.NewTransactionProc("READWRITETX", () => new ReadWriteTxn(), new RespCommandsInfo { Arity = 4 }); server.Register.NewProcedure("SUM", () => new Sum()); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/Resp/ACL/SetUserTests.cs b/test/Garnet.test/Resp/ACL/SetUserTests.cs index 669e233531..519a607c6c 100644 --- a/test/Garnet.test/Resp/ACL/SetUserTests.cs +++ b/test/Garnet.test/Resp/ACL/SetUserTests.cs @@ -20,11 +20,11 @@ class SetUserTests : AclTest /// Tests that new connections start with default user when no users are defined /// [Test] - public void PasswordlessDefaultUserTest() + public async Task PasswordlessDefaultUserTest() { // Create a new test server without password - should automatically login default user server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); // Check user is authenticated as default using var lightClientRequest = TestUtils.CreateRequest(); @@ -44,7 +44,7 @@ public async Task ProtectedDefaultUserErrorHandlingTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, defaultPassword: DummyPassword); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -69,7 +69,7 @@ public async Task ProtectedDefaultUserLoginImplicitTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, defaultPassword: DummyPassword); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -91,7 +91,7 @@ public async Task ProtectedDefaultUserLoginExplicitTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true, defaultPassword: DummyPassword); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -114,7 +114,7 @@ public async Task EnableAndDisableUsers() { // Create a new test server without default password server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -169,7 +169,7 @@ public async Task AddPasswordFromCleartextTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -199,7 +199,7 @@ public async Task AddPasswordFromHashTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -229,7 +229,7 @@ public async Task RemovePasswordFromCleartextTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -261,7 +261,7 @@ public async Task RemovePasswordFromHashTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -293,7 +293,7 @@ public async Task AddDuplicatePasswordTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -326,7 +326,7 @@ public async Task PasswordlessUserTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -349,7 +349,7 @@ public async Task ResetPasswordsTest() { // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -393,7 +393,7 @@ public async Task AddAndRemoveCategoryTest() // Create a new test server with password - should disallow any operation server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -449,7 +449,7 @@ public async Task ResetUser() { // Create a new test server server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -494,7 +494,7 @@ public async Task BadInputEmpty() { // Create a new test server server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -519,7 +519,7 @@ public async Task BadInputUnknownOperation() { // Create a new test server server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); @@ -544,7 +544,7 @@ public async Task KeyPatternsWildcard() { // Create a new test server server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useAcl: true); - server.Start(); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); diff --git a/test/Garnet.test/Resp/GarnetAuthenticatorTests.cs b/test/Garnet.test/Resp/GarnetAuthenticatorTests.cs index af92c5dd47..23fb94b80c 100644 --- a/test/Garnet.test/Resp/GarnetAuthenticatorTests.cs +++ b/test/Garnet.test/Resp/GarnetAuthenticatorTests.cs @@ -4,6 +4,7 @@ using System; using System.Text; using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using Garnet.server.Auth; using Garnet.server.Auth.Settings; @@ -77,8 +78,8 @@ public async Task InvalidatingAuthorizationAsync() return true; }; - using GarnetServer server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, authenticationSettings: authSettings); - server.Start(); + using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, authenticationSettings: authSettings); + await server.RunAsync(); using var c = TestUtils.GetGarnetClientSession(); c.Connect(); diff --git a/test/Garnet.test/RespAdminCommandsTests.cs b/test/Garnet.test/RespAdminCommandsTests.cs index 7bf4f07f06..eccfdcbabe 100644 --- a/test/Garnet.test/RespAdminCommandsTests.cs +++ b/test/Garnet.test/RespAdminCommandsTests.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -17,20 +18,22 @@ namespace Garnet.test [TestFixture] public class RespAdminCommandsTests { - GarnetServer server; + GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } @@ -157,13 +160,13 @@ public void SeSaveTest() } [Test] - public void SeSaveRecoverTest([Values] bool disableObj, [Values] bool useAzure) + public async Task SeSaveRecoverTest([Values] bool disableObj, [Values] bool useAzure) { if (useAzure) TestUtils.IgnoreIfNotRunningAzureTests(); - server.Dispose(); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: disableObj, UseAzureStorage: useAzure); - server.Start(); + _ = server.StartAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -176,9 +179,9 @@ public void SeSaveRecoverTest([Values] bool disableObj, [Values] bool useAzure) while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, UseAzureStorage: useAzure); - server.Start(); + _ = server.StartAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -189,7 +192,7 @@ public void SeSaveRecoverTest([Values] bool disableObj, [Values] bool useAzure) } [Test] - public void SeSaveRecoverObjectTest() + public async Task SeSaveRecoverObjectTest() { var key = "SeSaveRecoverTestObjectKey"; var ldata = new RedisValue[] { "a", "b", "c", "d" }; @@ -208,9 +211,9 @@ public void SeSaveRecoverObjectTest() while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true); - server.Start(); + _ = server.StartAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -223,7 +226,7 @@ public void SeSaveRecoverObjectTest() } [Test] - public void SeSaveRecoverCustomObjectTest() + public async Task SeSaveRecoverCustomObjectTest() { string key = "key"; string field = "field1"; @@ -247,11 +250,11 @@ public void SeSaveRecoverCustomObjectTest() while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true); server.Register.NewCommand("MYDICTSET", CommandType.ReadModifyWrite, factory, new MyDictSet(), new RespCommandsInfo { Arity = 4 }); server.Register.NewCommand("MYDICTGET", CommandType.Read, factory, new MyDictGet(), new RespCommandsInfo { Arity = 3 }); - server.Start(); + _ = server.StartAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -262,7 +265,7 @@ public void SeSaveRecoverCustomObjectTest() } [Test] - public void SeSaveRecoverCustomScriptTest() + public async Task SeSaveRecoverCustomScriptTest() { static void ValidateServerData(IDatabase db, string strKey, string strValue, string listKey, string listValue) { @@ -293,10 +296,10 @@ static void ValidateServerData(IDatabase db, string strKey, string strValue, str while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true); server.Register.NewProcedure("SETMAINANDOBJECT", () => new SetStringAndList()); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -309,13 +312,13 @@ static void ValidateServerData(IDatabase db, string strKey, string strValue, str [TestCase(63, 2, 1)] [TestCase(16, 16, 1)] [TestCase(5, 64, 1)] - public void SeSaveRecoverMultipleObjectsTest(int memorySize, int recoveryMemorySize, int pageSize) + public async Task SeSaveRecoverMultipleObjectsTest(int memorySize, int recoveryMemorySize, int pageSize) { string sizeToString(int size) => size + "k"; server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, MemorySize: sizeToString(memorySize), PageSize: sizeToString(pageSize)); - server.Start(); + await server.RunAsync(); var ldata = new RedisValue[] { "a", "b", "c", "d" }; var ldataArr = ldata.Select(x => x).Reverse().ToArray(); @@ -334,9 +337,9 @@ public void SeSaveRecoverMultipleObjectsTest(int memorySize, int recoveryMemoryS while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, lowMemory: true, MemorySize: sizeToString(recoveryMemorySize), PageSize: sizeToString(pageSize), objectStoreHeapMemorySize: "64k"); - server.Start(); + await server.RunAsync(); ClassicAssert.LessOrEqual(server.Provider.StoreWrapper.objectStore.MaxAllocatedPageCount, (recoveryMemorySize / pageSize) + 1); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) @@ -358,13 +361,13 @@ public void SeSaveRecoverMultipleObjectsTest(int memorySize, int recoveryMemoryS [TestCase("16k", "16k")] [TestCase("5k", "8k")] [TestCase("5k", "64k")] - public void SeSaveRecoverMultipleKeysTest(string memorySize, string recoveryMemorySize) + public async Task SeSaveRecoverMultipleKeysTest(string memorySize, string recoveryMemorySize) { bool disableObj = true; server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: disableObj, lowMemory: true, MemorySize: memorySize, PageSize: "512", enableAOF: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -401,9 +404,9 @@ public void SeSaveRecoverMultipleKeysTest(string memorySize, string recoveryMemo db.Execute("COMMITAOF"); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: disableObj, tryRecover: true, lowMemory: true, MemorySize: recoveryMemorySize, PageSize: "512", enableAOF: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -417,11 +420,11 @@ public void SeSaveRecoverMultipleKeysTest(string memorySize, string recoveryMemo } [Test] - public void SeAofRecoverTest() + public async Task SeAofRecoverTest() { - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableAOF: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -431,9 +434,9 @@ public void SeAofRecoverTest() db.Execute("COMMITAOF"); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableAOF: true, tryRecover: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { diff --git a/test/Garnet.test/RespAofAzureTests.cs b/test/Garnet.test/RespAofAzureTests.cs index 9787ba1e74..7a273cf688 100644 --- a/test/Garnet.test/RespAofAzureTests.cs +++ b/test/Garnet.test/RespAofAzureTests.cs @@ -3,6 +3,8 @@ using System; using System.Threading; +using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -12,7 +14,7 @@ namespace Garnet.test [TestFixture] public class RespAofAzureTests { - GarnetServer server; + GarnetApplication server; static readonly SortedSetEntry[] entries = [ new SortedSetEntry("a", 1), @@ -28,10 +30,10 @@ public class RespAofAzureTests ]; [SetUp] - public void Setup() + public async Task Setup() { server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableAOF: true, lowMemory: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); } [TearDown] @@ -41,7 +43,7 @@ public void TearDown() } [Test] - public void AofUpsertStoreRecoverTest() + public async Task AofUpsertStoreRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -51,9 +53,9 @@ public void AofUpsertStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -66,7 +68,7 @@ public void AofUpsertStoreRecoverTest() } [Test] - public void AofUpsertStoreAutoCommitRecoverTest() + public async Task AofUpsertStoreAutoCommitRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -76,9 +78,9 @@ public void AofUpsertStoreAutoCommitRecoverTest() } server.Store.WaitForCommit(); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -91,11 +93,11 @@ public void AofUpsertStoreAutoCommitRecoverTest() } [Test] - public void AofUpsertStoreAutoCommitCommitWaitRecoverTest() + public async Task AofUpsertStoreAutoCommitCommitWaitRecoverTest() { - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: false, enableAOF: true, commitWait: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -103,9 +105,10 @@ public void AofUpsertStoreAutoCommitCommitWaitRecoverTest() db.StringSet("SeAofUpsertRecoverTestKey1", "SeAofUpsertRecoverTestValue1"); db.StringSet("SeAofUpsertRecoverTestKey2", "SeAofUpsertRecoverTestValue2"); } - server.Dispose(false); + + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -118,7 +121,7 @@ public void AofUpsertStoreAutoCommitCommitWaitRecoverTest() } [Test] - public void AofUpsertStoreCkptRecoverTest() + public async Task AofUpsertStoreCkptRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -136,9 +139,9 @@ public void AofUpsertStoreCkptRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -153,7 +156,7 @@ public void AofUpsertStoreCkptRecoverTest() } [Test] - public void AofRMWStoreRecoverTest() + public async Task AofRMWStoreRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -163,9 +166,9 @@ public void AofRMWStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -178,7 +181,7 @@ public void AofRMWStoreRecoverTest() } [Test] - public void AofDeleteStoreRecoverTest() + public async Task AofDeleteStoreRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -198,9 +201,9 @@ public void AofDeleteStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -215,7 +218,7 @@ public void AofDeleteStoreRecoverTest() } [Test] - public void AofExpiryRMWStoreRecoverTest() + public async Task AofExpiryRMWStoreRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -233,9 +236,9 @@ public void AofExpiryRMWStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -248,7 +251,7 @@ public void AofExpiryRMWStoreRecoverTest() } [Test] - public void AofRMWObjectStoreRecoverTest() + public async Task AofRMWObjectStoreRecoverTest() { var key = "AofRMWObjectStoreRecoverTestKey"; @@ -267,9 +270,9 @@ public void AofRMWObjectStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -285,7 +288,7 @@ public void AofRMWObjectStoreRecoverTest() } [Test] - public void AofDeleteObjectStoreRecoverTest() + public async Task AofDeleteObjectStoreRecoverTest() { var key1 = "AofDeleteObjectStoreRecoverTestKey1"; var key2 = "AofDeleteObjectStoreRecoverTestKey2"; @@ -313,9 +316,9 @@ public void AofDeleteObjectStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -328,7 +331,7 @@ public void AofDeleteObjectStoreRecoverTest() } [Test] - public void AofRMWObjectStoreCopyUpdateRecoverTest() + public async Task AofRMWObjectStoreCopyUpdateRecoverTest() { var key = "AofRMWObjectStoreRecoverTestKey"; @@ -349,9 +352,9 @@ public void AofRMWObjectStoreCopyUpdateRecoverTest() db.SortedSetAdd("AofRMWObjectStoreRecoverTestKey" + 1, newEntries); } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -367,7 +370,7 @@ public void AofRMWObjectStoreCopyUpdateRecoverTest() } [Test] - public void AofMultiRMWStoreCkptRecoverTest() + public async Task AofMultiRMWStoreCkptRecoverTest() { long ret = 0; using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) @@ -404,9 +407,9 @@ public void AofMultiRMWStoreCkptRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true, UseAzureStorage: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { diff --git a/test/Garnet.test/RespAofTests.cs b/test/Garnet.test/RespAofTests.cs index 9cae04773e..6712bec043 100644 --- a/test/Garnet.test/RespAofTests.cs +++ b/test/Garnet.test/RespAofTests.cs @@ -5,7 +5,10 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using System.Threading.Tasks; +using Garnet.host; using Garnet.server; +using Microsoft.Extensions.Hosting; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -15,7 +18,7 @@ namespace Garnet.test [TestFixture] public class RespAofTests { - GarnetServer server; + GarnetApplication server; private IReadOnlyDictionary respCustomCommandsInfo; static readonly SortedSetEntry[] entries = @@ -33,13 +36,13 @@ public class RespAofTests ]; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); ClassicAssert.IsTrue(TestUtils.TryGetCustomCommandsInfo(out respCustomCommandsInfo)); ClassicAssert.IsNotNull(respCustomCommandsInfo); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableAOF: true, lowMemory: true); - server.Start(); + await server.RunAsync(); } [TearDown] @@ -50,7 +53,7 @@ public void TearDown() } [Test] - public void AofUpsertStoreRecoverTest() + public async Task AofUpsertStoreRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -60,7 +63,7 @@ public void AofUpsertStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -75,7 +78,7 @@ public void AofUpsertStoreRecoverTest() } [Test] - public void AofUpsertStoreAutoCommitRecoverTest() + public async Task AofUpsertStoreAutoCommitRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -85,7 +88,7 @@ public void AofUpsertStoreAutoCommitRecoverTest() } server.Store.WaitForCommit(); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -101,11 +104,11 @@ public void AofUpsertStoreAutoCommitRecoverTest() [Test] [CancelAfter(10_000)] - public void AofUpsertStoreCommitTaskRecoverTest() + public async Task AofUpsertStoreCommitTaskRecoverTest() { - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: false, enableAOF: true, commitFrequencyMs: 100); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -115,7 +118,7 @@ public void AofUpsertStoreCommitTaskRecoverTest() } server.Store.WaitForCommit(); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -130,9 +133,9 @@ public void AofUpsertStoreCommitTaskRecoverTest() } [Test] - public void AofUpsertStoreAutoCommitCommitWaitRecoverTest() + public async Task AofUpsertStoreAutoCommitCommitWaitRecoverTest() { - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: false, enableAOF: true, commitWait: true); server.Start(); @@ -142,7 +145,7 @@ public void AofUpsertStoreAutoCommitCommitWaitRecoverTest() db.StringSet("SeAofUpsertRecoverTestKey1", "SeAofUpsertRecoverTestValue1"); db.StringSet("SeAofUpsertRecoverTestKey2", "SeAofUpsertRecoverTestValue2"); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -157,9 +160,9 @@ public void AofUpsertStoreAutoCommitCommitWaitRecoverTest() } [Test] - public void AofTransactionStoreAutoCommitCommitWaitRecoverTest() + public async Task AofTransactionStoreAutoCommitCommitWaitRecoverTest() { - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: false, enableAOF: true, commitWait: true); server.Start(); @@ -172,7 +175,7 @@ public void AofTransactionStoreAutoCommitCommitWaitRecoverTest() ClassicAssert.IsTrue(transaction.Execute()); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -187,7 +190,7 @@ public void AofTransactionStoreAutoCommitCommitWaitRecoverTest() } [Test] - public void AofUpsertStoreCkptRecoverTest() + public async Task AofUpsertStoreCkptRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -205,7 +208,7 @@ public void AofUpsertStoreCkptRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -222,7 +225,7 @@ public void AofUpsertStoreCkptRecoverTest() } [Test] - public void AofRMWStoreRecoverTest() + public async Task AofRMWStoreRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -232,7 +235,7 @@ public void AofRMWStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -247,7 +250,7 @@ public void AofRMWStoreRecoverTest() } [Test] - public void AofDeleteStoreRecoverTest() + public async Task AofDeleteStoreRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -267,7 +270,7 @@ public void AofDeleteStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -284,7 +287,7 @@ public void AofDeleteStoreRecoverTest() } [Test] - public void AofExpiryRMWStoreRecoverTest() + public async Task AofExpiryRMWStoreRecoverTest() { using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -302,7 +305,7 @@ public void AofExpiryRMWStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -317,7 +320,7 @@ public void AofExpiryRMWStoreRecoverTest() } [Test] - public void AofRMWObjectStoreRecoverTest() + public async Task AofRMWObjectStoreRecoverTest() { var key = "AofRMWObjectStoreRecoverTestKey"; @@ -336,7 +339,7 @@ public void AofRMWObjectStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -354,7 +357,7 @@ public void AofRMWObjectStoreRecoverTest() } [Test] - public void AofDeleteObjectStoreRecoverTest() + public async Task AofDeleteObjectStoreRecoverTest() { var key1 = "AofDeleteObjectStoreRecoverTestKey1"; var key2 = "AofDeleteObjectStoreRecoverTestKey2"; @@ -382,7 +385,7 @@ public void AofDeleteObjectStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -397,7 +400,7 @@ public void AofDeleteObjectStoreRecoverTest() } [Test] - public void AofRMWObjectStoreCopyUpdateRecoverTest() + public async Task AofRMWObjectStoreCopyUpdateRecoverTest() { var key = "AofRMWObjectStoreRecoverTestKey"; @@ -418,7 +421,7 @@ public void AofRMWObjectStoreCopyUpdateRecoverTest() db.SortedSetAdd("AofRMWObjectStoreRecoverTestKey" + 1, newEntries); } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -436,7 +439,7 @@ public void AofRMWObjectStoreCopyUpdateRecoverTest() } [Test] - public void AofUpsertObjectStoreRecoverTest() + public async Task AofUpsertObjectStoreRecoverTest() { var origList = new RedisValue[] { "a", "b", "c", "d" }; var key1 = "lkey1"; @@ -461,7 +464,7 @@ public void AofUpsertObjectStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Start(); @@ -475,16 +478,16 @@ public void AofUpsertObjectStoreRecoverTest() } [Test] - public void AofUpsertCustomObjectRecoverTest() + public async Task AofUpsertCustomObjectRecoverTest() { - void RegisterCustomCommand(GarnetServer gServer) + void RegisterCustomCommand(GarnetApplication gServer) { var factory = new MyDictFactory(); gServer.Register.NewCommand("MYDICTSET", CommandType.ReadModifyWrite, factory, new MyDictSet(), respCustomCommandsInfo["MYDICTSET"]); gServer.Register.NewCommand("MYDICTGET", CommandType.Read, factory, new MyDictGet(), respCustomCommandsInfo["MYDICTGET"]); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableAOF: true); RegisterCustomCommand(server); server.Start(); @@ -511,10 +514,10 @@ void RegisterCustomCommand(GarnetServer gServer) } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); RegisterCustomCommand(server); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -526,7 +529,7 @@ void RegisterCustomCommand(GarnetServer gServer) } [Test] - public void AofUpsertCustomScriptRecoverTest() + public async Task AofUpsertCustomScriptRecoverTest() { static void ValidateServerData(IDatabase db, string strKey, string strValue, string listKey, string listValue) { @@ -537,7 +540,7 @@ static void ValidateServerData(IDatabase db, string strKey, string strValue, str ClassicAssert.AreEqual(listValue, (string)retList[0]); } - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableAOF: true); server.Register.NewProcedure("SETMAINANDOBJECT", () => new SetStringAndList()); server.Start(); @@ -555,10 +558,10 @@ static void ValidateServerData(IDatabase db, string strKey, string strValue, str } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Register.NewProcedure("SETMAINANDOBJECT", () => new SetStringAndList()); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -567,7 +570,7 @@ static void ValidateServerData(IDatabase db, string strKey, string strValue, str } [Test] - public void AofMultiRMWStoreCkptRecoverTest() + public async Task AofMultiRMWStoreCkptRecoverTest() { long ret = 0; using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) @@ -604,9 +607,9 @@ public void AofMultiRMWStoreCkptRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { @@ -617,7 +620,7 @@ public void AofMultiRMWStoreCkptRecoverTest() } [Test] - public void AofListObjectStoreRecoverTest() + public async Task AofListObjectStoreRecoverTest() { var key = "AofListObjectStoreRecoverTest"; var ldata = new RedisValue[] { "a", "b", "c", "d" }; @@ -635,9 +638,9 @@ public void AofListObjectStoreRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { @@ -649,7 +652,7 @@ public void AofListObjectStoreRecoverTest() } [Test] - public void AofCustomTxnRecoverTest() + public async Task AofCustomTxnRecoverTest() { server.Register.NewTransactionProc("READWRITETX", () => new ReadWriteTxn(), new RespCommandsInfo { Arity = 4 }); string readkey = "readme"; @@ -669,11 +672,11 @@ public void AofCustomTxnRecoverTest() } server.Store.CommitAOF(true); - server.Dispose(false); + await server.StopAsync(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); server.Register.NewTransactionProc("READWRITETX", () => new ReadWriteTxn(), new RespCommandsInfo { Arity = 4 }); - server.Start(); + await server.RunAsync(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) { diff --git a/test/Garnet.test/RespBlockingListTests.cs b/test/Garnet.test/RespBlockingListTests.cs index 2e92524161..5ba9a27d62 100644 --- a/test/Garnet.test/RespBlockingListTests.cs +++ b/test/Garnet.test/RespBlockingListTests.cs @@ -4,6 +4,7 @@ using System; using System.Text; using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,21 +13,21 @@ namespace Garnet.test { public class RespBlockingListTests { - GarnetServer server; + GarnetApplication server; private TaskFactory taskFactory = new(); [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespCommandTests.cs b/test/Garnet.test/RespCommandTests.cs index 8059ee3c25..919ecc5463 100644 --- a/test/Garnet.test/RespCommandTests.cs +++ b/test/Garnet.test/RespCommandTests.cs @@ -7,7 +7,9 @@ using System.Linq; using System.Reflection; using System.Runtime.InteropServices; +using System.Threading.Tasks; using Garnet.common; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -21,7 +23,7 @@ namespace Garnet.test [TestFixture] public class RespCommandTests { - GarnetServer server; + GarnetApplication server; private string extTestDir; private IReadOnlyDictionary respCommandsInfo; private IReadOnlyDictionary respSubCommandsInfo; @@ -31,7 +33,7 @@ public class RespCommandTests private IReadOnlyDictionary respCustomCommandsDocs; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); extTestDir = Path.Combine(TestUtils.MethodTestDir, "test"); @@ -47,13 +49,13 @@ public void Setup() ClassicAssert.IsNotNull(respCustomCommandsDocs); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, disablePubSub: true, extensionBinPaths: [extTestDir]); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); TestUtils.DeleteDirectory(Directory.GetParent(extTestDir)?.FullName); } diff --git a/test/Garnet.test/RespCustomCommandTests.cs b/test/Garnet.test/RespCustomCommandTests.cs index cef9353517..f40a67f091 100644 --- a/test/Garnet.test/RespCustomCommandTests.cs +++ b/test/Garnet.test/RespCustomCommandTests.cs @@ -12,6 +12,7 @@ using System.Threading; using System.Threading.Tasks; using Garnet.common; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -144,12 +145,12 @@ public override bool Execute(TGarnetApi garnetApi, ref CustomProcedu [TestFixture] public class RespCustomCommandTests { - GarnetServer server; + GarnetApplication server; private string _extTestDir1; private string _extTestDir2; [SetUp] - public void Setup() + public async Task Setup() { _extTestDir1 = Path.Combine(TestUtils.MethodTestDir, "test1"); _extTestDir2 = Path.Combine(TestUtils.MethodTestDir, "test2"); @@ -159,13 +160,13 @@ public void Setup() disablePubSub: true, extensionBinPaths: [_extTestDir1, _extTestDir2], extensionAllowUnsignedAssemblies: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); TestUtils.DeleteDirectory(Directory.GetParent(_extTestDir1)?.FullName); } diff --git a/test/Garnet.test/RespGetLowMemoryTests.cs b/test/Garnet.test/RespGetLowMemoryTests.cs index ff39389a7e..46b9c4b297 100644 --- a/test/Garnet.test/RespGetLowMemoryTests.cs +++ b/test/Garnet.test/RespGetLowMemoryTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -14,22 +15,22 @@ namespace Garnet.test [TestFixture] public class RespGetLowMemoryTests { - GarnetServer server; + GarnetApplication server; Random r; [SetUp] - public void Setup() + public async Task Setup() { r = new Random(335); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, getSG: true, disablePubSub: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespHashTests.cs b/test/Garnet.test/RespHashTests.cs index e4a7b9b8d2..3d944d0939 100644 --- a/test/Garnet.test/RespHashTests.cs +++ b/test/Garnet.test/RespHashTests.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,22 +16,22 @@ namespace Garnet.test [TestFixture] public class RespHashTests { - GarnetServer server; + GarnetApplication server; static readonly HashEntry[] entries = new HashEntry[100]; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespInfoTests.cs b/test/Garnet.test/RespInfoTests.cs index c6c5e077f9..d242725d48 100644 --- a/test/Garnet.test/RespInfoTests.cs +++ b/test/Garnet.test/RespInfoTests.cs @@ -4,6 +4,8 @@ using System; using System.Linq; using System.Threading; +using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -13,22 +15,22 @@ namespace Garnet.test [TestFixture] public class RespInfoTests { - GarnetServer server; + GarnetApplication server; Random r; [SetUp] - public void Setup() + public async Task Setup() { r = new Random(674386); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, disablePubSub: true, latencyMonitor: true, metricsSamplingFreq: 1, DisableObjects: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespListGarnetClientTests.cs b/test/Garnet.test/RespListGarnetClientTests.cs index 44137a37e3..9cb5b2e4a5 100644 --- a/test/Garnet.test/RespListGarnetClientTests.cs +++ b/test/Garnet.test/RespListGarnetClientTests.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using Garnet.client; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,14 +13,14 @@ namespace Garnet.test [TestFixture] public class RespListGarnetClientTests { - private GarnetServer server; + private GarnetApplication server; [OneTimeSetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true); - server.Start(); + await server.RunAsync(); } private static object[] LeftPushTestCases = @@ -271,9 +272,9 @@ private static async Task ValidateListContentAsync(GarnetClient db, string key, } [OneTimeTearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespListTests.cs b/test/Garnet.test/RespListTests.cs index fbf140c13d..9e865496e0 100644 --- a/test/Garnet.test/RespListTests.cs +++ b/test/Garnet.test/RespListTests.cs @@ -7,6 +7,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -17,22 +18,22 @@ namespace Garnet.test [TestFixture] class RespListTests { - GarnetServer server; + GarnetApplication server; Random r; [SetUp] - public void Setup() + public async Task Setup() { r = new Random(674386); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespLowMemoryTests.cs b/test/Garnet.test/RespLowMemoryTests.cs index 091d100637..534d2a33a9 100644 --- a/test/Garnet.test/RespLowMemoryTests.cs +++ b/test/Garnet.test/RespLowMemoryTests.cs @@ -2,6 +2,8 @@ // Licensed under the MIT license. using System; +using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -11,20 +13,20 @@ namespace Garnet.test [TestFixture] public class RespLowMemoryTests { - GarnetServer server; + GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespMetricsTest.cs b/test/Garnet.test/RespMetricsTest.cs index 024cd94954..cb76891192 100644 --- a/test/Garnet.test/RespMetricsTest.cs +++ b/test/Garnet.test/RespMetricsTest.cs @@ -4,7 +4,9 @@ using System; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Garnet.common; +using Garnet.host; using Microsoft.Extensions.Logging; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,14 +17,14 @@ namespace Garnet.test [TestFixture] public class RespMetricsTest { - GarnetServer server; + GarnetApplication server; ILoggerFactory loggerFactory; Random r; - private void StartServer(int metricsSamplingFreq = -1, bool latencyMonitor = false) + private async Task StartServer(int metricsSamplingFreq = -1, bool latencyMonitor = false) { server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, metricsSamplingFreq: metricsSamplingFreq, latencyMonitor: latencyMonitor); - server.Start(); + await server.RunAsync(); } [SetUp] @@ -35,9 +37,9 @@ public void Setup() } [TearDown] - public void TearDown() + public async Task TearDown() { - server?.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); } diff --git a/test/Garnet.test/RespModuleTests.cs b/test/Garnet.test/RespModuleTests.cs index ae88521a2c..7125a95afd 100644 --- a/test/Garnet.test/RespModuleTests.cs +++ b/test/Garnet.test/RespModuleTests.cs @@ -4,6 +4,8 @@ using System.IO; using System.Reflection; using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -13,11 +15,11 @@ namespace Garnet.test [TestFixture] public class RespModuleTests { - GarnetServer server; + GarnetApplication server; private string testModuleDir; [SetUp] - public void Setup() + public async Task Setup() { testModuleDir = Path.Combine(TestUtils.MethodTestDir, "testModules"); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); @@ -25,13 +27,13 @@ public void Setup() disablePubSub: true, extensionBinPaths: [testModuleDir], extensionAllowUnsignedAssemblies: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); TestUtils.DeleteDirectory(Directory.GetParent(testModuleDir)?.FullName); } @@ -174,7 +176,7 @@ public void TestModuleLoad() [Test] - public void TestModuleLoadUsingGarnetOptions() + public async Task TestModuleLoadUsingGarnetOptions() { var onLoad = @"context.Initialize(""TestModule1"", 1); @@ -194,7 +196,7 @@ public void TestModuleLoadUsingGarnetOptions() server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, disablePubSub: true, loadModulePaths: [module1Path, module2Path]); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); diff --git a/test/Garnet.test/RespPubSubTests.cs b/test/Garnet.test/RespPubSubTests.cs index 35b1d63168..a6c7ed0951 100644 --- a/test/Garnet.test/RespPubSubTests.cs +++ b/test/Garnet.test/RespPubSubTests.cs @@ -5,6 +5,8 @@ using System.Linq; using System.Security.Cryptography; using System.Threading; +using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -14,20 +16,20 @@ namespace Garnet.test [TestFixture] class RespPubSubTests { - GarnetServer server; + GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, pubSubPageSize: "256k"); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespScanCommandsTests.cs b/test/Garnet.test/RespScanCommandsTests.cs index bba4077d18..40437d3c32 100644 --- a/test/Garnet.test/RespScanCommandsTests.cs +++ b/test/Garnet.test/RespScanCommandsTests.cs @@ -5,6 +5,8 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,23 +17,23 @@ namespace Garnet.test [TestFixture] public class RespScanCommandsTests { - GarnetServer server; + GarnetApplication server; private IReadOnlyDictionary respCustomCommandsInfo; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); ClassicAssert.IsTrue(TestUtils.TryGetCustomCommandsInfo(out respCustomCommandsInfo)); ClassicAssert.IsNotNull(respCustomCommandsInfo); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespSetTest.cs b/test/Garnet.test/RespSetTest.cs index 59344b462d..222ae83fa9 100644 --- a/test/Garnet.test/RespSetTest.cs +++ b/test/Garnet.test/RespSetTest.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -17,20 +18,20 @@ namespace Garnet.test [TestFixture] public class RespSetTest { - GarnetServer server; + GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespSortedSetGarnetClientTests.cs b/test/Garnet.test/RespSortedSetGarnetClientTests.cs index ef7d456381..39af4ef36e 100644 --- a/test/Garnet.test/RespSortedSetGarnetClientTests.cs +++ b/test/Garnet.test/RespSortedSetGarnetClientTests.cs @@ -12,6 +12,7 @@ using Garnet.client; using Garnet.client.GarnetClientAPI; using Garnet.common; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -21,7 +22,7 @@ namespace Garnet.test [TestFixture] public class RespSortedSetGarnetClientTests { - protected GarnetServer server; + protected GarnetApplication server; ManualResetEventSlim waiter; const int maxIterations = 3; @@ -41,19 +42,19 @@ public class RespSortedSetGarnetClientTests [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, enableAOF: true); - server.Start(); + await server.RunAsync(); waiter = new ManualResetEventSlim(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespSortedSetGeoTests.cs b/test/Garnet.test/RespSortedSetGeoTests.cs index b7c6d1cc88..284dfe9164 100644 --- a/test/Garnet.test/RespSortedSetGeoTests.cs +++ b/test/Garnet.test/RespSortedSetGeoTests.cs @@ -5,7 +5,9 @@ using System.Globalization; using System.Linq; using System.Text; +using System.Threading.Tasks; using Garnet.common; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -16,7 +18,7 @@ namespace Garnet.test [TestFixture] public class RespSortedSetGeoTests { - GarnetServer server; + GarnetApplication server; readonly string[,] cities = new string[,] { {"-74.0059413", "40.7127837", "New York"}, {"-118.2436849", "34.0522342", "Los Angeles"}, @@ -126,17 +128,17 @@ public class RespSortedSetGeoTests [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespSortedSetTests.cs b/test/Garnet.test/RespSortedSetTests.cs index 3abfaa67c2..66f61b9250 100644 --- a/test/Garnet.test/RespSortedSetTests.cs +++ b/test/Garnet.test/RespSortedSetTests.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Embedded.server; using Garnet.common; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -27,7 +28,7 @@ namespace Garnet.test [TestFixture] public class RespSortedSetTests { - protected GarnetServer server; + protected GarnetApplication server; static readonly SortedSetEntry[] entries = [ @@ -73,18 +74,18 @@ public class RespSortedSetTests [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs index 86c0adfa96..5a2a402e85 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/Garnet.test/RespTests.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Garnet.client; using Garnet.common; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -20,22 +21,22 @@ namespace Garnet.test [TestFixture] public class RespTests { - GarnetServer server; + GarnetApplication server; Random r; [SetUp] - public void Setup() + public async Task Setup() { r = new Random(674386); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, disablePubSub: false); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } @@ -1053,13 +1054,13 @@ public void SingleDelete() } [Test] - public void SingleDeleteWithObjectStoreDisabled() + public async Task SingleDeleteWithObjectStoreDisabled() { TearDown(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); - server.Start(); + await server.RunAsync(); var key = "delKey"; var value = "1234"; @@ -1085,13 +1086,13 @@ private string GetRandomString(int len) } [Test] - public void SingleDeleteWithObjectStoreDisable_LTM() + public async Task SingleDeleteWithObjectStoreDisable_LTM() { TearDown(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, DisableObjects: true); - server.Start(); + await server.RunAsync(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -1125,14 +1126,14 @@ public void SingleDeleteWithObjectStoreDisable_LTM() } [Test] - public void MultiKeyDelete([Values] bool withoutObjectStore) + public async Task MultiKeyDelete([Values] bool withoutObjectStore) { if (withoutObjectStore) { TearDown(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); - server.Start(); + await server.RunAsync(); } using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); @@ -1193,14 +1194,14 @@ public void MultiKeyDeleteObjectStore() } [Test] - public void MultiKeyUnlink([Values] bool withoutObjectStore) + public async Task MultiKeyUnlink([Values] bool withoutObjectStore) { if (withoutObjectStore) { TearDown(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); - server.Start(); + await server.RunAsync(); } using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); @@ -1259,14 +1260,14 @@ public void MultiKeyUnlinkObjectStore() } [Test] - public void SingleExists([Values] bool withoutObjectStore) + public async Task SingleExists([Values] bool withoutObjectStore) { if (withoutObjectStore) { TearDown(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); - server.Start(); + await server.RunAsync(); } using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -1524,14 +1525,14 @@ public void SingleRenameWithExpiry() } [Test] - public void SingleRenameKeyEdgeCase([Values] bool withoutObjectStore) + public async Task SingleRenameKeyEdgeCase([Values] bool withoutObjectStore) { if (withoutObjectStore) { TearDown(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); - server.Start(); + await server.RunAsync(); } using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -3382,13 +3383,13 @@ public void HelloTest1() } [Test] - public void AsyncTest1() + public async Task AsyncTest1() { // Set up low-memory database TearDown(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, DisableObjects: true); - server.Start(); + await server.RunAsync(); string firstKey = null, firstValue = null, lastKey = null, lastValue = null; diff --git a/test/Garnet.test/RespTlsTests.cs b/test/Garnet.test/RespTlsTests.cs index 7e61240516..5cca1d7d81 100644 --- a/test/Garnet.test/RespTlsTests.cs +++ b/test/Garnet.test/RespTlsTests.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Threading.Tasks; using Garnet.common; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -17,20 +18,20 @@ namespace Garnet.test [TestFixture] public class RespTlsTests { - GarnetServer server; + GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, EnableTLS: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/RespTransactionProcTests.cs b/test/Garnet.test/RespTransactionProcTests.cs index b3f3c05c29..d32e597081 100644 --- a/test/Garnet.test/RespTransactionProcTests.cs +++ b/test/Garnet.test/RespTransactionProcTests.cs @@ -2,6 +2,8 @@ // Licensed under the MIT license. using System.Threading; +using System.Threading.Tasks; +using Garnet.host; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,20 +14,20 @@ namespace Garnet.test [TestFixture] public class RespTransactionProcTests { - GarnetServer server; + GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, disablePubSub: true); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } diff --git a/test/Garnet.test/TestUtils.cs b/test/Garnet.test/TestUtils.cs index 7e9698a494..890c5bd4c2 100644 --- a/test/Garnet.test/TestUtils.cs +++ b/test/Garnet.test/TestUtils.cs @@ -184,7 +184,7 @@ internal static void IgnoreIfNotRunningAzureTests() /// /// Create GarnetServer /// - public static GarnetServer CreateGarnetServer( + public static GarnetApplication CreateGarnetServer( string logCheckpointDir, bool disablePubSub = false, bool tryRecover = false, @@ -342,7 +342,7 @@ public static GarnetServer CreateGarnetServer( var app = builder.Build(); - return app.Services.GetRequiredService(); + return app; } else { @@ -350,7 +350,7 @@ public static GarnetServer CreateGarnetServer( var app = builder.Build(); - return app.Services.GetRequiredService(); + return app; } } diff --git a/test/Garnet.test/TransactionTests.cs b/test/Garnet.test/TransactionTests.cs index 76e91799b1..f1d1620b54 100644 --- a/test/Garnet.test/TransactionTests.cs +++ b/test/Garnet.test/TransactionTests.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; +using Garnet.host; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -12,29 +13,29 @@ namespace Garnet.test [TestFixture] public class TransactionTests { - GarnetServer server; + GarnetApplication server; [SetUp] - public void Setup() + public async Task Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); - server.Start(); + await server.RunAsync(); } [TearDown] - public void TearDown() + public async Task TearDown() { - server.Dispose(); + await server.StopAsync(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir); } - public void SetUpWithLowMemory() + public async Task SetUpWithLowMemory() { TearDown(); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true); - server.Start(); + await server.RunAsync(); } [Test]