From 6123b08bda25b2ba7a3ca4e726d0d972b792d985 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Wed, 9 Jun 2021 13:29:57 -0700 Subject: [PATCH 01/11] FO 3.1.13 --- Build-SFPkgs.ps1 | 8 +-- Documentation/Using.md | 2 +- .../Utilities/Logger.cs | 54 ++++++++------ .../Utilities/NetworkUsage.cs | 65 ++++++++--------- .../Telemetry}/ServiceEventSource.cs | 72 +++++++++++++++---- .../Utilities/Telemetry/TelemetryData.cs | 7 ++ FabricObserver.nuspec.template | 7 +- FabricObserver/Program.cs | 11 ++- README.md | 10 +-- .../SampleObserverPlugin.csproj | 2 +- 10 files changed, 149 insertions(+), 89 deletions(-) rename {FabricObserver => FabricObserver.Extensibility/Utilities/Telemetry}/ServiceEventSource.cs (80%) diff --git a/Build-SFPkgs.ps1 b/Build-SFPkgs.ps1 index 640999af..93adbd6e 100644 --- a/Build-SFPkgs.ps1 +++ b/Build-SFPkgs.ps1 @@ -23,11 +23,11 @@ function Build-SFPkg { try { Push-Location $scriptPath - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.SelfContained.3.1.12" "$scriptPath\bin\release\FabricObserver\linux-x64\self-contained\FabricObserverType" - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.FrameworkDependent.3.1.12" "$scriptPath\bin\release\FabricObserver\linux-x64\framework-dependent\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.SelfContained.3.1.13" "$scriptPath\bin\release\FabricObserver\linux-x64\self-contained\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.FrameworkDependent.3.1.13" "$scriptPath\bin\release\FabricObserver\linux-x64\framework-dependent\FabricObserverType" - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.1.12" "$scriptPath\bin\release\FabricObserver\win-x64\self-contained\FabricObserverType" - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.FrameworkDependent.3.1.12" "$scriptPath\bin\release\FabricObserver\win-x64\framework-dependent\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.1.13" "$scriptPath\bin\release\FabricObserver\win-x64\self-contained\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.FrameworkDependent.3.1.13" "$scriptPath\bin\release\FabricObserver\win-x64\framework-dependent\FabricObserverType" } finally { Pop-Location diff --git a/Documentation/Using.md b/Documentation/Using.md index 97b63fee..eac64708 100644 --- a/Documentation/Using.md +++ b/Documentation/Using.md @@ -539,7 +539,7 @@ $appParams = @{ "FabricSystemObserverEnabled" = "true"; "FabricSystemObserverMem Then execute the application upgrade with ```Powershell -Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.1.12 -ApplicationParameter $appParams -Monitored -FailureAction rollback +Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.1.13 -ApplicationParameter $appParams -Monitored -FailureAction rollback ``` Note: On *Linux*, this will restart FO processes (one at a time, UD Walk with safety checks) due to the way Linux Capabilites work. In a nutshell, for any kind of application upgrade, we have to re-run the FO setup script to get the Capabilities in place. For Windows, FO processes will NOT be restarted. diff --git a/FabricObserver.Extensibility/Utilities/Logger.cs b/FabricObserver.Extensibility/Utilities/Logger.cs index 6f72f99e..3ceeffa6 100644 --- a/FabricObserver.Extensibility/Utilities/Logger.cs +++ b/FabricObserver.Extensibility/Utilities/Logger.cs @@ -4,11 +4,12 @@ // ------------------------------------------------------------ using System; -using System.Diagnostics.Tracing; using System.IO; using System.Runtime.InteropServices; using System.Threading; using FabricObserver.Observers.Interfaces; +using FabricObserver.Observers.Utilities.Telemetry; +using Newtonsoft.Json; using NLog; using NLog.Config; using NLog.Targets; @@ -24,7 +25,7 @@ public sealed class Logger : IObserverLogger private const int Retries = 5; // This needs to be static to prevent internal EventSource instantiation errors. - private static EventSource etwLogger; + //private static EventSource etwLogger; private readonly string loggerName; @@ -44,19 +45,6 @@ private string Filename get; } - private EventSource EtwLogger - { - get - { - if (EnableETWLogging && etwLogger == null) - { - etwLogger = new EventSource(ObserverConstants.EventSourceProviderName); - } - - return etwLogger; - } - } - public bool EnableETWLogging { get; set; @@ -143,9 +131,37 @@ public void LogWarning(string format, params object[] parameters) OLogger.Warn(format, parameters); } + /// + /// Logs EventSource events and automatically determines Level based on object (T data) content inspection. + /// + /// Anonymous/generic type. + /// Name of the event. + /// Anonymous object instance. public void LogEtw(string eventName, T data) { - EtwLogger?.Write(eventName, data); + if (!EnableETWLogging) + { + return; + } + + // All FO ETW events are written as anonymous .NET types (anonymous object intances with fields/properties) + // housed in FabricObserverDataEvent events of FabricObserverETWProvider EventSource provider. This means they + // are JSON serializable for use in content inspection. + string s = JsonConvert.SerializeObject(data); + + if (!string.IsNullOrWhiteSpace(s) && s.Contains("Warning")) + { + ServiceEventSource.Current.DataTypeWriteWarning(eventName, data); + return; + } + + if (!string.IsNullOrWhiteSpace(s) && s.Contains("Error")) + { + ServiceEventSource.Current.DataTypeWriteError(eventName, data); + return; + } + + ServiceEventSource.Current.DataTypeWriteInfo(eventName, data); } public bool TryWriteLogFile(string path, string content) @@ -172,11 +188,9 @@ public bool TryWriteLogFile(string path, string content) File.WriteAllText(path, content); return true; } - catch (IOException) - { - } - catch (UnauthorizedAccessException) + catch (Exception e) when (e is ArgumentException || e is IOException || e is UnauthorizedAccessException) { + } Thread.Sleep(1000); diff --git a/FabricObserver.Extensibility/Utilities/NetworkUsage.cs b/FabricObserver.Extensibility/Utilities/NetworkUsage.cs index e24cfd72..c42af8d1 100644 --- a/FabricObserver.Extensibility/Utilities/NetworkUsage.cs +++ b/FabricObserver.Extensibility/Utilities/NetworkUsage.cs @@ -23,51 +23,45 @@ public static (int LowPort, int HighPort) TupleGetFabricApplicationPortRangeForN return (-1, -1); } - StringReader sreader = null; - XmlReader xreader = null; - try { - // Safe XML pattern - *Do not use LoadXml*. - var xdoc = new XmlDocument { XmlResolver = null }; - sreader = new StringReader(clusterManifestXml); - xreader = XmlReader.Create(sreader, new XmlReaderSettings { XmlResolver = null }); - xdoc.Load(xreader); + using (var sreader = new StringReader(clusterManifestXml)) + { + using (var xreader = XmlReader.Create(sreader, new XmlReaderSettings { XmlResolver = null })) + { + // Safe XML pattern - *Do not use LoadXml*. + var xdoc = new XmlDocument { XmlResolver = null }; + xdoc.Load(xreader); - // Cluster Information. - var nsmgr = new XmlNamespaceManager(xdoc.NameTable); - nsmgr.AddNamespace("sf", "http://schemas.microsoft.com/2011/01/fabric"); + // Cluster Information. + var nsmgr = new XmlNamespaceManager(xdoc.NameTable); + nsmgr.AddNamespace("sf", "http://schemas.microsoft.com/2011/01/fabric"); - // SF Application Port Range. - var applicationEndpointsNode = xdoc.SelectSingleNode($"//sf:NodeTypes//sf:NodeType[@Name='{nodeType}']//sf:ApplicationEndpoints", nsmgr); + // SF Application Port Range. + var applicationEndpointsNode = xdoc.SelectSingleNode($"//sf:NodeTypes//sf:NodeType[@Name='{nodeType}']//sf:ApplicationEndpoints", nsmgr); - if (applicationEndpointsNode == null) - { - return (-1, -1); - } + if (applicationEndpointsNode == null) + { + return (-1, -1); + } - var ret = (int.Parse(applicationEndpointsNode.Attributes?.Item(0)?.Value ?? "-1"), - int.Parse(applicationEndpointsNode.Attributes?.Item(1)?.Value ?? "-1")); + var ret = (int.Parse(applicationEndpointsNode.Attributes?.Item(0)?.Value ?? "-1"), + int.Parse(applicationEndpointsNode.Attributes?.Item(1)?.Value ?? "-1")); - return ret; + return ret; + } + } } catch (Exception e) when (e is ArgumentException || e is NullReferenceException || e is XmlException) { // continue } - finally - { - sreader?.Dispose(); - xreader?.Dispose(); - } return (-1, -1); } public static int GetActiveFirewallRulesCount() { - ManagementObjectCollection results = null; - ManagementObjectSearcher searcher = null; int count = -1; // This method is not implemented for Linux yet. @@ -80,17 +74,18 @@ public static int GetActiveFirewallRulesCount() { var scope = new ManagementScope("\\\\.\\ROOT\\StandardCimv2"); var q = new ObjectQuery("SELECT * FROM MSFT_NetFirewallRule WHERE Enabled=1"); - searcher = new ManagementObjectSearcher(scope, q); - results = searcher.Get(); - count = results.Count; + + using (var searcher = new ManagementObjectSearcher(scope, q)) + { + using (var results = searcher.Get()) + { + count = results.Count; + } + } } catch (ManagementException) { - } - finally - { - results?.Dispose(); - searcher?.Dispose(); + } return count; diff --git a/FabricObserver/ServiceEventSource.cs b/FabricObserver.Extensibility/Utilities/Telemetry/ServiceEventSource.cs similarity index 80% rename from FabricObserver/ServiceEventSource.cs rename to FabricObserver.Extensibility/Utilities/Telemetry/ServiceEventSource.cs index c07f4299..b82ab121 100644 --- a/FabricObserver/ServiceEventSource.cs +++ b/FabricObserver.Extensibility/Utilities/Telemetry/ServiceEventSource.cs @@ -9,23 +9,23 @@ using System.Threading.Tasks; using Microsoft.ServiceFabric.TelemetryLib; -namespace FabricObserver +namespace FabricObserver.Observers.Utilities.Telemetry { - [EventSource(Name = "Service-Fabric-FabricObserver", Guid = "373f2a64-4823-518a-32d1-78e36f922c24")] - internal sealed class ServiceEventSource : EventSource, ITelemetryEventSource + public sealed class ServiceEventSource : EventSource, ITelemetryEventSource { public static readonly ServiceEventSource Current = new ServiceEventSource(); static ServiceEventSource() { // A workaround for the problem where ETW activities do not get tracked until Tasks infrastructure is initialized. - // This problem will be fixed in .NET Framework 4.6.2. + // This problem is fixed in .NET Framework 4.6.2. If you are running this version or greater, then delete the below code. _ = Task.Run(() => { }); } // Instance constructor is private to enforce singleton semantics - private ServiceEventSource() + private ServiceEventSource() : base(ObserverConstants.EventSourceProviderName) { + } // Define an instance method for each event you want to record and apply an [Event] attribute to it. @@ -47,6 +47,48 @@ public void Message(string message, params object[] args) Message(finalMessage); } + [NonEvent] + public void DataTypeWriteInfo(string eventName, T data) + { + var options = new EventSourceOptions + { + ActivityOptions = EventActivityOptions.None, + Keywords = Keywords.ResourceUsage, + Opcode = EventOpcode.Info, + Level = EventLevel.Verbose, + }; + + Write(eventName, options, data); + } + + [NonEvent] + public void DataTypeWriteWarning(string eventName, T data) + { + var options = new EventSourceOptions + { + ActivityOptions = EventActivityOptions.None, + Keywords = Keywords.ErrorOrWarning, + Opcode = EventOpcode.Info, + Level = EventLevel.Warning, + }; + + Write(eventName, options, data); + } + + [NonEvent] + public void DataTypeWriteError(string eventName, T data) + { + var options = new EventSourceOptions + { + ActivityOptions = EventActivityOptions.None, + Keywords = Keywords.ErrorOrWarning, + Opcode = EventOpcode.Info, + Level = EventLevel.Error, + }; + + Write(eventName, options, data); + } + private const int MessageEventId = 1; [Event(MessageEventId, Level = EventLevel.Informational, Message = "{0}")] @@ -180,19 +222,19 @@ public void VerboseMessage(string message, params object[] args) "fabricObserverConfiguration = {2}, " + "fabricObserverHealthState = {3}")] public void FabricObserverRuntimeNodeEvent( - string clusterId, - string applicationVersion, - string foConfigInfo, - string foHealthInfo) + string clusterId, + string applicationVersion, + string foConfigInfo, + string foHealthInfo) { if (IsEnabled()) { WriteEvent( - FabricObserverTelemetryEventId, - clusterId, - applicationVersion, - foConfigInfo, - foHealthInfo); + FabricObserverTelemetryEventId, + clusterId, + applicationVersion, + foConfigInfo, + foHealthInfo); } } @@ -217,6 +259,8 @@ public static class Keywords { public const EventKeywords Requests = (EventKeywords)0x1L; public const EventKeywords ServiceInitialization = (EventKeywords)0x2L; + public const EventKeywords ResourceUsage = (EventKeywords)0x4L; + public const EventKeywords ErrorOrWarning = (EventKeywords)0x8L; } } } \ No newline at end of file diff --git a/FabricObserver.Extensibility/Utilities/Telemetry/TelemetryData.cs b/FabricObserver.Extensibility/Utilities/Telemetry/TelemetryData.cs index 7993d6de..743cd065 100644 --- a/FabricObserver.Extensibility/Utilities/Telemetry/TelemetryData.cs +++ b/FabricObserver.Extensibility/Utilities/Telemetry/TelemetryData.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Threading; using Microsoft.ServiceFabric.TelemetryLib; +using Newtonsoft.Json; namespace FabricObserver.Observers.Utilities.Telemetry { @@ -98,6 +99,12 @@ public object Value get; set; } + [JsonConstructor] + public TelemetryData() + { + + } + public TelemetryData(FabricClient fabricClient, CancellationToken cancellationToken) { var (clusterId, _, _) = ClusterIdentificationUtility.TupleGetClusterIdAndTypeAsync(fabricClient, cancellationToken).Result; diff --git a/FabricObserver.nuspec.template b/FabricObserver.nuspec.template index b4fe7b8e..7dab41f4 100644 --- a/FabricObserver.nuspec.template +++ b/FabricObserver.nuspec.template @@ -2,9 +2,12 @@ %PACKAGE_ID% - 3.1.12 + 3.1.13 - Performance optimizations - most changes impact private workingset. This is a performance-focused release with one bug fix (regression) in OSObserver Windows hotfix data for telemetry. + - New EventSource implementation to support ETW event listening impls (important for those who want to implement EventListener-derived types in plugins to capture FO ETW event data). + - ServiceEventSource class modified with new functions, moved to Extensibility library. + - Changes to Logger and TelemetryData classes to support new ServiceEventSource impl. + - NetworkUsage class code change: replaced try-finally explicit disposal pattern with using blocks. Microsoft MIT diff --git a/FabricObserver/Program.cs b/FabricObserver/Program.cs index efedb40a..90663c2a 100644 --- a/FabricObserver/Program.cs +++ b/FabricObserver/Program.cs @@ -7,6 +7,7 @@ using System.Diagnostics; using System.Threading; using Microsoft.ServiceFabric.Services.Runtime; +using FabricObserver.Observers.Utilities.Telemetry; namespace FabricObserver { @@ -19,19 +20,15 @@ private static void Main() { try { - ServiceRuntime.RegisterServiceAsync( - "FabricObserverType", - context => new FabricObserver(context)).GetAwaiter().GetResult(); - - ServiceEventSource.Current.ServiceTypeRegistered(Process.GetCurrentProcess().Id, nameof(FabricObserver)); + ServiceRuntime.RegisterServiceAsync("FabricObserverType", context => new FabricObserver(context)).GetAwaiter().GetResult(); + ServiceEventSource.Current?.ServiceTypeRegistered(Process.GetCurrentProcess().Id, nameof(FabricObserver)); // Prevents this host process from terminating so services keep running. Thread.Sleep(Timeout.Infinite); } catch (Exception e) { - ServiceEventSource.Current.ServiceHostInitializationFailed(e.ToString()); - + ServiceEventSource.Current?.ServiceHostInitializationFailed(e.ToString()); throw; } } diff --git a/README.md b/README.md index 9bba9ed9..5f1269b6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# FabricObserver 3.1.12 +# FabricObserver 3.1.13 [**FabricObserver (FO)**](https://github.com/microsoft/service-fabric-observer/releases) is a complete implementation of a generic resource usage watchdog service written as a stateless, singleton Service Fabric .NET Core 3.1 application that 1. Monitors a broad range of resources that tend to be important to all Service Fabric applications, like disk, CPU, memory, networking, and cluster certificates out-of-the-box. @@ -135,19 +135,19 @@ Connect-ServiceFabricCluster -ConnectionEndpoint @('sf-win-cluster.westus2.cloud #Copy $path contents (FO app package) to server: -Copy-ServiceFabricApplicationPackage -ApplicationPackagePath $path -CompressPackage -ApplicationPackagePathInImageStore FO3112 -TimeoutSec 1800 +Copy-ServiceFabricApplicationPackage -ApplicationPackagePath $path -CompressPackage -ApplicationPackagePathInImageStore FO3113 -TimeoutSec 1800 #Register FO ApplicationType: -Register-ServiceFabricApplicationType -ApplicationPathInImageStore FO3112 +Register-ServiceFabricApplicationType -ApplicationPathInImageStore FO3113 #Create FO application (if not already deployed at lesser version): -New-ServiceFabricApplication -ApplicationName fabric:/FabricObserver -ApplicationTypeName FabricObserverType -ApplicationTypeVersion 3.1.12 +New-ServiceFabricApplication -ApplicationName fabric:/FabricObserver -ApplicationTypeName FabricObserverType -ApplicationTypeVersion 3.1.13 #OR if updating existing version: -Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.1.12 -Monitored -FailureAction rollback +Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.1.13 -Monitored -FailureAction rollback ``` ## Observer Model diff --git a/SampleObserverPlugin/SampleObserverPlugin.csproj b/SampleObserverPlugin/SampleObserverPlugin.csproj index 68ed90fb..2ccd6579 100644 --- a/SampleObserverPlugin/SampleObserverPlugin.csproj +++ b/SampleObserverPlugin/SampleObserverPlugin.csproj @@ -17,6 +17,6 @@ - + From e9eda7998fd15989e43b0527424b145313de92a1 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Wed, 9 Jun 2021 13:34:10 -0700 Subject: [PATCH 02/11] FO 3.1.13 update nuspec. --- FabricObserver.nuspec.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FabricObserver.nuspec.template b/FabricObserver.nuspec.template index 7dab41f4..11f9694b 100644 --- a/FabricObserver.nuspec.template +++ b/FabricObserver.nuspec.template @@ -4,9 +4,9 @@ %PACKAGE_ID% 3.1.13 - - New EventSource implementation to support ETW event listening impls (important for those who want to implement EventListener-derived types in plugins to capture FO ETW event data). + - New EventSource impl to better support ETW event listening impls (important for plugins that need to capture FO ETW event data). - ServiceEventSource class modified with new functions, moved to Extensibility library. - - Changes to Logger and TelemetryData classes to support new ServiceEventSource impl. + - Changes to Logger (removed EtwLogger EventSource) and TelemetryData classes to support new ServiceEventSource impl. - NetworkUsage class code change: replaced try-finally explicit disposal pattern with using blocks. Microsoft From ebe6620f4786868322755ac0151ba0d332e19e8d Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Wed, 9 Jun 2021 14:21:48 -0700 Subject: [PATCH 03/11] FO 3.1.13 update manifest versions --- FabricObserver/PackageRoot/ServiceManifest._linux.xml | 8 ++++---- FabricObserver/PackageRoot/ServiceManifest.xml | 8 ++++---- .../ApplicationPackageRoot/ApplicationManifest.xml | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/FabricObserver/PackageRoot/ServiceManifest._linux.xml b/FabricObserver/PackageRoot/ServiceManifest._linux.xml index eaf8adf0..782d1edc 100644 --- a/FabricObserver/PackageRoot/ServiceManifest._linux.xml +++ b/FabricObserver/PackageRoot/ServiceManifest._linux.xml @@ -1,6 +1,6 @@  @@ -11,7 +11,7 @@ - + setcaps.sh @@ -27,11 +27,11 @@ - + - + \ No newline at end of file diff --git a/FabricObserver/PackageRoot/ServiceManifest.xml b/FabricObserver/PackageRoot/ServiceManifest.xml index 6b2328f1..2edde408 100644 --- a/FabricObserver/PackageRoot/ServiceManifest.xml +++ b/FabricObserver/PackageRoot/ServiceManifest.xml @@ -1,6 +1,6 @@  @@ -11,7 +11,7 @@ - + FabricObserver @@ -21,11 +21,11 @@ - + - + \ No newline at end of file diff --git a/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml b/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml index 2af47686..b5032906 100644 --- a/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml +++ b/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml @@ -1,5 +1,5 @@  - + - + From 05a46e4895e58145bbcedc110a2b8c7379be505c Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Wed, 9 Jun 2021 14:23:56 -0700 Subject: [PATCH 04/11] FO 3.1.13 update .gitignore... --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 84f6ede7..42a0c818 100644 --- a/.gitignore +++ b/.gitignore @@ -335,3 +335,4 @@ ASALocalRun/ **/PublishProfiles/Cloud.xml /FabricObserver/observer_logs /FabricObserver/PackageRoot/Data/Plugins/SampleNewObserver.dll +/nuget.exe From d751cda551b6d3eacb4d5bbd1bacb78f8265c26b Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Wed, 9 Jun 2021 14:55:09 -0700 Subject: [PATCH 05/11] FO 3.1.13 Test comments.. --- FabricObserverTests/ObserverTest.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FabricObserverTests/ObserverTest.cs b/FabricObserverTests/ObserverTest.cs index 79f1a041..760c119b 100644 --- a/FabricObserverTests/ObserverTest.cs +++ b/FabricObserverTests/ObserverTest.cs @@ -336,7 +336,8 @@ public async Task Successful_CertificateObserver_Run_Cancellation_Via_ObserverMa Assert.IsFalse(obsMgr.IsObserverRunning); } - /* NOTE: These tests are flaky. Run them one by one or, even better, make them better. */ + /* NOTE: The below tests are *flaky* (not the feature they test, though... :-). + * Run them one by one or, even better, make them better.. */ [TestMethod] public async Task Successful_AppObserver_Run_Cancellation_Via_ObserverManager() From af65c462d21d46d548bab6efa26fc8f6b6e991b5 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Thu, 10 Jun 2021 16:56:24 -0700 Subject: [PATCH 06/11] FO 3.1.3 - Update to FRUD --- .../Utilities/FabricResourceUsageData.cs | 14 +++++++------- FabricObserver/Observers/ObserverManager.cs | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/FabricObserver.Extensibility/Utilities/FabricResourceUsageData.cs b/FabricObserver.Extensibility/Utilities/FabricResourceUsageData.cs index 8977a0ad..e4afae77 100644 --- a/FabricObserver.Extensibility/Utilities/FabricResourceUsageData.cs +++ b/FabricObserver.Extensibility/Utilities/FabricResourceUsageData.cs @@ -99,7 +99,7 @@ public IList Data private bool isInWarningState; /// - /// Gets count of warnings per observer instance across iterations for the lifetime of the Observer. + /// Gets count of total warnings for the lifetime of this instance. /// public int LifetimeWarningCount { @@ -144,8 +144,8 @@ public double AverageDataValue } /// - /// Gets or sets a value indicating whether there is an active warning state on this instance. - /// Set to false when warning state changes to Ok. + /// Gets or sets a value indicating whether there is an active Error or Warning health state on this instance. + /// Set to false when health state changes to Ok. /// public bool ActiveErrorOrWarning { @@ -192,13 +192,13 @@ public bool IsUnhealthy(TU threshold) public T StandardDeviation => Data?.Count > 0 ? Statistics.StandardDeviation(Data) : default; /// - /// Gets SlidingWindow Max: A sorted list of sliding window maximums. + /// Gets SlidingWindow Max: A sorted list of sliding window maximums. This is only availabe when Data is CircularBufferCollection. /// - public IList SlidingWindowMax => Data?.Count > 0 ? Statistics.SlidingWindow(Data, 3, WindowType.Max) : new List(1); + public IList SlidingWindowMax => Data?.Count >= 3 ? Statistics.SlidingWindow(Data, 3, WindowType.Max) : null; /// - /// Gets SlidingWindow Min: A sorted list of sliding window minimums. + /// Gets SlidingWindow Min: A sorted list of sliding window minimums. This is only availabe when Data is CircularBufferCollection. /// - public IList SlidingWindowMin => Data?.Count > 0 ? Statistics.SlidingWindow(Data, 3, WindowType.Min) : new List(1); + public IList SlidingWindowMin => Data?.Count >= 3 ? Statistics.SlidingWindow(Data, 3, WindowType.Min) : null; } } \ No newline at end of file diff --git a/FabricObserver/Observers/ObserverManager.cs b/FabricObserver/Observers/ObserverManager.cs index 24b9b9ce..026b2d34 100644 --- a/FabricObserver/Observers/ObserverManager.cs +++ b/FabricObserver/Observers/ObserverManager.cs @@ -209,10 +209,10 @@ public ObserverManager(IServiceProvider serviceProvider, FabricClient fabricClie } string codePkgVersion = FabricServiceContext.CodePackageActivationContext.CodePackageVersion; - string serviceManifestVersion = FabricServiceContext.CodePackageActivationContext - .GetConfigurationPackageObject("Config").Description.ServiceManifestVersion; + string serviceManifestVersion = FabricServiceContext.CodePackageActivationContext.GetConfigurationPackageObject("Config").Description.ServiceManifestVersion; string filepath = Path.Combine(Logger.LogFolderBasePath, - $"fo_telemetry_sent_{codePkgVersion.Replace(".", string.Empty)}_{serviceManifestVersion.Replace(".", string.Empty)}_{FabricServiceContext.NodeContext.NodeType}.log"); + $"fo_telemetry_sent_{codePkgVersion.Replace(".", string.Empty)}_" + + $"{serviceManifestVersion.Replace(".", string.Empty)}_{FabricServiceContext.NodeContext.NodeType}.log"); #if !DEBUG // If this has already been sent for this activated version (code/config) of node type x @@ -222,16 +222,16 @@ public ObserverManager(IServiceProvider serviceProvider, FabricClient fabricClie } #endif var telemetryEvents = new TelemetryEvents( - FabricClientInstance, - FabricServiceContext, - ServiceEventSource.Current, - this.token); + FabricClientInstance, + FabricServiceContext, + ServiceEventSource.Current, + this.token); if (telemetryEvents.FabricObserverRuntimeNodeEvent(codePkgVersion, GetFabricObserverInternalConfiguration(), "HealthState.Initialized")) { // Log a file to prevent re-sending this in case of process restart(s). - // This non-PII FO/Cluster info is versioned and should only be sent once per deployment (config or code updates.). + // This non-PII FO/Cluster info is versioned and should only be sent once per deployment (config or code updates). _ = Logger.TryWriteLogFile(filepath, GetFabricObserverInternalConfiguration()); } } From 59b8984d630a28b8b50582155d392586b37496f5 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Fri, 11 Jun 2021 10:53:05 -0700 Subject: [PATCH 07/11] FO 3.1.13 Log file management updates, AppObs logging (handled ex). --- .../Utilities/DataTableFileLogger.cs | 7 ++-- .../Utilities/Logger.cs | 34 ++++++++++++++++++- FabricObserver/Observers/AppObserver.cs | 16 +++------ FabricObserverTests/ObserverTest.cs | 2 +- 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/FabricObserver.Extensibility/Utilities/DataTableFileLogger.cs b/FabricObserver.Extensibility/Utilities/DataTableFileLogger.cs index 698e7e8b..a7acdea6 100644 --- a/FabricObserver.Extensibility/Utilities/DataTableFileLogger.cs +++ b/FabricObserver.Extensibility/Utilities/DataTableFileLogger.cs @@ -94,8 +94,8 @@ public void ConfigureLogger(string filename) var csvPath = Path.Combine(logFullPath, filename + ".csv"); - // Clean out old files if written as MultipleFilesNoArchives. - if (MaxArchiveCsvFileLifetimeDays > 0 && FileWriteFormat == CsvFileWriteFormat.MultipleFilesNoArchives) + // Clean out old files. + if (MaxArchiveCsvFileLifetimeDays > 0) { TryCleanLogFolder(logFullPath, TimeSpan.FromDays(MaxArchiveCsvFileLifetimeDays)); } @@ -162,8 +162,7 @@ public static void Flush() private static void TryCleanLogFolder(string folderPath, TimeSpan maxAge) { - if (!Directory.Exists(folderPath) || - DateTime.UtcNow.Subtract(Directory.GetLastWriteTimeUtc(folderPath)) < maxAge) + if (!Directory.Exists(folderPath)) { return; } diff --git a/FabricObserver.Extensibility/Utilities/Logger.cs b/FabricObserver.Extensibility/Utilities/Logger.cs index 3ceeffa6..f8134f97 100644 --- a/FabricObserver.Extensibility/Utilities/Logger.cs +++ b/FabricObserver.Extensibility/Utilities/Logger.cs @@ -268,7 +268,7 @@ private void InitializeLoggers() LogFolderBasePath = logFolderBase; string file = Path.Combine(logFolderBase, "fabric_observer.log"); - if (!string.IsNullOrEmpty(FolderName) && !string.IsNullOrEmpty(Filename)) + if (!string.IsNullOrWhiteSpace(FolderName) && !string.IsNullOrWhiteSpace(Filename)) { string folderPath = Path.Combine(logFolderBase, FolderName); file = Path.Combine(folderPath, Filename); @@ -276,6 +276,13 @@ private void InitializeLoggers() FilePath = file; + // Clean out old log files. This is to ensure the supplied policy is enforced if FO is restarted before the MaxArchiveFileLifetimeDays has been reached. + // This is because Logger FileTarget settings are not preserved across FO deployments. + if (MaxArchiveFileLifetimeDays > 0) + { + TryCleanLogFolder(Path.Combine(logFolderBase, FolderName), TimeSpan.FromDays(MaxArchiveFileLifetimeDays)); + } + var targetName = loggerName + "LogFile"; if (LogManager.Configuration == null) @@ -311,5 +318,30 @@ private void InitializeLoggers() TimeSource.Current = new AccurateUtcTimeSource(); OLogger = LogManager.GetLogger(loggerName); } + + private static void TryCleanLogFolder(string folderPath, TimeSpan maxAge) + { + if (!Directory.Exists(folderPath)) + { + return; + } + + string[] files = Directory.GetFiles(folderPath, "*", SearchOption.AllDirectories); + + foreach (string file in files) + { + try + { + if (DateTime.UtcNow.Subtract(File.GetCreationTime(file)) >= maxAge) + { + File.Delete(file); + } + } + catch (Exception e) when (e is ArgumentException || e is IOException || e is UnauthorizedAccessException) + { + + } + } + } } } \ No newline at end of file diff --git a/FabricObserver/Observers/AppObserver.cs b/FabricObserver/Observers/AppObserver.cs index 5e097504..efb826fd 100644 --- a/FabricObserver/Observers/AppObserver.cs +++ b/FabricObserver/Observers/AppObserver.cs @@ -777,21 +777,13 @@ private async Task MonitorDeployedAppsAsync(CancellationToken token) } catch (Exception e) when (e is ArgumentException || e is InvalidOperationException || e is Win32Exception) { -#if DEBUG - WriteToLogWithLevel( - ObserverName, - $"MonitorDeployedAppsAsync: failed to find current service process or target process " + - $"is running at a higher privilege than FO for " + - $"{repOrInst.ApplicationName?.OriginalString ?? repOrInst.ApplicationTypeName}{Environment.NewLine}{e}", - LogLevel.Warning); -#endif + ObserverLogger.LogWarning( + $"Handled exception in MonitorDeployedAppsAsync: process {processId} is not running or it's running at a higher privilege than FabricObserver.{Environment.NewLine}" + + $"ServiceName: {repOrInst.ServiceName?.OriginalString ?? "unknown"}{Environment.NewLine}Error message: {e.Message}"); } catch (Exception e) when (!(e is OperationCanceledException || e is TaskCanceledException)) { - WriteToLogWithLevel( - ObserverName, - $"Unhandled exception in MonitorDeployedAppsAsync:{Environment.NewLine}{e}", - LogLevel.Warning); + ObserverLogger.LogWarning($"Unhandled exception in MonitorDeployedAppsAsync:{Environment.NewLine}{e}"); // Fix the bug.. throw; diff --git a/FabricObserverTests/ObserverTest.cs b/FabricObserverTests/ObserverTest.cs index 760c119b..13ac023c 100644 --- a/FabricObserverTests/ObserverTest.cs +++ b/FabricObserverTests/ObserverTest.cs @@ -1542,7 +1542,7 @@ private static async Task CleanupTestHealthReportsAsync(ObserverBase obs = null) // App reports if (obs is {HasActiveFabricErrorOrWarning: true} && obs.ObserverName != ObserverConstants.NetworkObserverName) { - if (obs.AppNames.Count > 0 && obs.AppNames.All(a => !string.IsNullOrEmpty(a) && a.Contains("fabric:/"))) + if (obs.AppNames.Count > 0 && obs.AppNames.All(a => !string.IsNullOrWhiteSpace(a) && a.Contains("fabric:/"))) { foreach (var app in obs.AppNames) { From 94bea7c5ae2163359c1f5f6e7f1470578bb97d72 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Mon, 14 Jun 2021 10:57:55 -0700 Subject: [PATCH 08/11] FO 3.1.3 - Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f1269b6..31ba6a69 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # FabricObserver 3.1.13 [**FabricObserver (FO)**](https://github.com/microsoft/service-fabric-observer/releases) is a complete implementation of a generic resource usage watchdog service written as a stateless, singleton Service Fabric .NET Core 3.1 application that -1. Monitors a broad range of resources that tend to be important to all Service Fabric applications, like disk, CPU, memory, networking, and cluster certificates out-of-the-box. +1. Monitors a broad range of resources that tend to be important to all Service Fabric applications, like disk use, CPU use, memory use, endpoint availability, ephemeral TCP port use, and app/cluster certificate health out-of-the-box. 2. Runs on multiple versions of Windows Server and Ubuntu 16.04 and 18.04 3. Provides [an easy-to-use extensibility model](/Documentation/Plugins.md) for creating [custom Observers](/SampleObserverPlugin) out of band (so, you don't need to clone the repo to build an Observer). See [ContainerObserver](https://github.com/GitTorre/ContainerObserver) for a complete plugin impl that extends FO with SF container app resource monitoring and alerting. 4. Supports [Configuration Setting Application Updates](/Documentation/Using.md#parameterUpdates) for any observer for any supported setting. From 6dc0f41b13a8d5ea073004c23133216625424987 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Mon, 14 Jun 2021 11:22:45 -0700 Subject: [PATCH 09/11] FO 3.1.13 - Observers.md --- Documentation/Observers.md | 246 ++++++++++++++----------------------- 1 file changed, 89 insertions(+), 157 deletions(-) diff --git a/Documentation/Observers.md b/Documentation/Observers.md index 59f4f529..6731003d 100644 --- a/Documentation/Observers.md +++ b/Documentation/Observers.md @@ -34,7 +34,13 @@ You can quickly get started by reading [this](/Documentation/Using.md). ***Note: All observers that monitor various resources output serialized instances of TelemetryData type (JSON). This JSON string is set as the Description property of a Health Event. This is done for a few reasons: Telemetry support and for consuming services that need to deserialize the data to inform some related workflow. In later versions of SF, SFX will display only the textual pieces of this serialized object instance, making it easier to read in SFX's Details view.*** - +``` +The vast majority of settings for any observer are provided in ApplicationManifest.xml +as required overridden parameters. For AppObserver and NetworkObserver, +all thresholds/settings are housed in json files, not XML. +For every other observer, it's XML as per usual. +``` + ## AppObserver Observer that monitors CPU usage, Memory use, and Port use for Service Fabric Application services (processes). This observer will alert (SF Health event) when user-supplied thresholds are reached. **Please note that this observer should not be used to monitor docker container applications. It is not designed for this task. Instead, please consider employing [ContainerObserver](https://github.com/GitTorre/ContainerObserver), which is designed specifically for container monitoring**. @@ -51,71 +57,21 @@ will be ignored (they are not omitted below so you can see what a fully specifie We recommend you omit all Error thresholds until you become more comfortable with the behavior of your services and the side effects they have on machine resources**. -Example JSON config file located in **PackageRoot\\Config** -folder (AppObserver.config.json): +Example JSON config file located in **PackageRoot\\Config** folder (AppObserver.config.json). This is an example of a configuration that applies +to all Service Fabric user (non-System) application service processes running on the virtual machine. ```JSON [ { - "targetApp": "fabric:/MyApp", - "cpuErrorLimitPercent": 0, - "dumpProcessOnError": false, - "memoryErrorLimitPercent": 0, - "memoryWarningLimitPercent": 60, - "networkErrorActivePorts": 0, - "networkErrorEphemeralPorts": 0, - "networkWarningActivePorts": 800, - "networkWarningEphemeralPorts": 400, - "errorOpenFileHandles": 0, - "warningOpenFileHandles": 5000 - }, - { - "targetApp": "fabric:/MyApp1", - "serviceIncludeList": "MyService42, MyOtherService42", - "cpuErrorLimitPercent": 0, - "cpuWarningLimitPercent": 8, - "dumpProcessOnError": false, - "memoryErrorLimitPercent": 0, - "memoryWarningLimitPercent": 60, - "networkErrorActivePorts": 0, - "networkWarningActivePorts": 800, - "networkErrorEphemeralPorts": 0, - "networkWarningEphemeralPorts": 400, - "errorOpenFileHandles": 0, - "warningOpenFileHandles": 2500 - }, - { - "targetApp": "fabric:/FabricObserver", - "cpuErrorLimitPercent": 0, - "cpuWarningLimitPercent": 30, - "dumpProcessOnError": false, - "memoryErrorLimitPercent": 0, - "memoryWarningLimitPercent": 30, - "networkErrorActivePorts": 0, - "networkWarningActivePorts": 800, - "networkErrorEphemeralPorts": 0, - "networkWarningEphemeralPorts": 400, - "errorOpenFileHandles": 0, - "warningOpenFileHandles": 1000 - }, - { - "targetApp": "fabric:/FabricObserverWebApi", - "cpuErrorLimitPercent": 0, - "cpuWarningLimitPercent": 30, - "dumpProcessOnError": false, - "memoryErrorLimitPercent": 0, - "memoryWarningLimitPercent": 30, - "networkErrorActivePorts": 0, - "networkWarningActivePorts": 800, - "networkErrorEphemeralPorts": 0, - "networkWarningEphemeralPorts": 400, - "errorOpenFileHandles": 0, - "warningOpenFileHandles": 1000 + "targetApp": "*", + "cpuWarningLimitPercent": 80, + "memoryWarningLimitMb": 1048, + "networkWarningEphemeralPorts": 7000 } ] ``` Settings descriptions: -All settings are optional, ***except target OR targetType***, and can be omitted if you don't want to track. Or, you can leave the values blank ("") or set to 0 for numeric values. For process memory use, you can supply either MB values (a la 1024 for 1GB) for Working Set (Private) or percentage of total memory in use by process (as an integer). +All settings are optional, ***except target OR targetType***, and can be omitted if you don't want to track. For process memory use, you can supply either MB values (a la 1024 for 1GB) for Working Set (Private) or percentage of total memory in use by process (as an integer, 1 - 100). | Setting | Description | | :--- | :--- | @@ -243,46 +199,43 @@ This observer monitors Fabric system service processes e.g., Fabric, FabricAppli FabricDCA, FabricDnsService, FabricFAS, FabricGateway, FabricHost, etc... **NOTE:** -Only enable FabricSystemObserver ***after*** you get a sense of what impact your services have on the SF runtime. +Only enable FabricSystemObserver ***after*** you get a sense of what impact your services have on the SF runtime, in particular Fabric.exe. This is very important because there is no "one threshold fits all" across warning/error thresholds for any of the SF system services. -That is, we (the SF team) do not have a fixed set of guaranteed problematic warning thresholds for SF infrastructure services. Your code can cause Fabric.exe to eat a lot of CPU, for example, but this is not a Fabric.exe bug. - -Again, it is best to Enable this observer only after you have done some experimentation with monitoring how your service code impacts Service Fabric system services. Otherwise, you may end up generating noise and creating support tickets when there is in fact nothing wrong with SF, but that your service code is just stressing SF services (e.g., Fabric.exe). This is of course useful to know, but FabricSystemObserver can't tell you that your code is the problem and we do not want you to create a support ticket because FSO warned you about something SF engineers can't fix for you... +By default, FabricObserver runs as NetworkUser on Windows and sfappsuser on Linux. These are non-privileged accounts and therefore for any service +running as System or root, default FabricObserver can't monitor process behavior (this is always true on Windows). That said, there are only a few system +services you would care about: Fabric.exe and FabricGateway.exe. Fabric.exe is generally the system service that your code can directly impact with respect to machine resource usage. -**Input**: Settings defined in Settings.xml with required overridable settings specified in ApplicationManifest.xml. +**Input - Settings.xml**: Only ClusterOperationTimeoutSeconds is set in Settings.xml. ```xml
- - - - - - - - - - - - - - - - - - - - - - + ...
``` + +**Input - ApplicationManifest.xml**: Threshold settings are defined (overriden) in ApplicationManifest.xml. + +```xml + + + + + + + + + + + + + + + + + +``` + **Output**: Log text(Error/Warning), Service Fabric Health Report (Error/Warning/Ok), ETW, Telemetry Example SFX output (Informational): @@ -331,44 +284,29 @@ Example NetworkObserver.config.json configuration: ```javascript [ { - "targetApp": "fabric:/MyApp", - "endpoints": [ - { - "hostname": "google.com", - "port": 443, - "protocol": "http" - }, - { - "hostname": "facebook.com", - "port": 443, - "protocol": "http" - }, - { - "hostname": "westusserver001.database.windows.net", - "port": 1433, - "protocol": "tcp" - } - ] + "targetApp": "fabric:/MyApp0", + "endpoints": [ + { + "hostname": "https://myazuresrvice42.westus2.cloudapp.azure.com", + "port": 443, + "protocol": "http" + }, + { + "hostname": "somesqlservername.database.windows.net", + "port": 1433, + "protocol": "tcp" + } + ] }, { - "targetApp": "fabric:/MyApp2", - "endpoints": [ - { - "hostname": "google.com", - "port": 443, - "protocol": "http" - }, - { - "hostname": "microsoft.com", - "port": 443, - "protocol": "http" - }, - { - "hostname": "eastusserver007.database.windows.net", - "port": 1433, - "protocol": "tcp" - } - ] + "targetApp": "fabric:/MyApp1", + "endpoints": [ + { + "hostname": "somesqlservername.database.windows.net", + "port": 1433, + "protocol": "tcp" + } + ] } ] ``` @@ -381,42 +319,36 @@ network failures which will result in Fabric Health warnings that live until the ## NodeObserver This observer monitors VM level resource usage across CPU, Memory, firewall rules, static and dynamic ports (aka ephemeral ports). - Thresholds for Erorr and Warning signals are user-supplied in PackageRoot/Config/Settings.xml. + Thresholds for Erorr and Warning signals are user-supplied in ApplicationManifest.xml. -**Input**: +**Input - ApplicationManifest.xml**: ```xml -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
+ + + + + + + + + + + + + + + + + + + + + + ``` + | Setting | Description | | :--- | :--- | -| **ClusterOperationTimeoutSeconds** | Number of seconds for timeout for making SF async API calls. | | **CpuErrorLimitPercent** | Maximum CPU percentage that should generate an Error | | **CpuWarningLimitPercent** | Minimum CPU percentage that should generate a Warning | | **EnableTelemetry** | Whether or not to send Observer data to diagnostics/log analytics service. | From 5dfab6ad84d45a7c04f59529ee3231b85e68332e Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Mon, 14 Jun 2021 11:26:30 -0700 Subject: [PATCH 10/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 31ba6a69..181c2e31 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # FabricObserver 3.1.13 [**FabricObserver (FO)**](https://github.com/microsoft/service-fabric-observer/releases) is a complete implementation of a generic resource usage watchdog service written as a stateless, singleton Service Fabric .NET Core 3.1 application that -1. Monitors a broad range of resources that tend to be important to all Service Fabric applications, like disk use, CPU use, memory use, endpoint availability, ephemeral TCP port use, and app/cluster certificate health out-of-the-box. +1. Monitors a broad range of machine resources that tend to be very important to all Service Fabric applications, like disk use (usage across IO and capacity/space), CPU use, memory use, endpoint availability, ephemeral TCP port use, and app/cluster certificate health out-of-the-box. 2. Runs on multiple versions of Windows Server and Ubuntu 16.04 and 18.04 3. Provides [an easy-to-use extensibility model](/Documentation/Plugins.md) for creating [custom Observers](/SampleObserverPlugin) out of band (so, you don't need to clone the repo to build an Observer). See [ContainerObserver](https://github.com/GitTorre/ContainerObserver) for a complete plugin impl that extends FO with SF container app resource monitoring and alerting. 4. Supports [Configuration Setting Application Updates](/Documentation/Using.md#parameterUpdates) for any observer for any supported setting. From c88cc2fca032503475bd056e21c4c4cb1639f9f0 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Mon, 14 Jun 2021 11:28:17 -0700 Subject: [PATCH 11/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 181c2e31..d463b5c1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # FabricObserver 3.1.13 [**FabricObserver (FO)**](https://github.com/microsoft/service-fabric-observer/releases) is a complete implementation of a generic resource usage watchdog service written as a stateless, singleton Service Fabric .NET Core 3.1 application that -1. Monitors a broad range of machine resources that tend to be very important to all Service Fabric applications, like disk use (usage across IO and capacity/space), CPU use, memory use, endpoint availability, ephemeral TCP port use, and app/cluster certificate health out-of-the-box. +1. Monitors a broad range of machine resources that tend to be very important to all Service Fabric applications, like disk space consumption, CPU use, memory use, endpoint availability, ephemeral TCP port use, and app/cluster certificate health out-of-the-box. 2. Runs on multiple versions of Windows Server and Ubuntu 16.04 and 18.04 3. Provides [an easy-to-use extensibility model](/Documentation/Plugins.md) for creating [custom Observers](/SampleObserverPlugin) out of band (so, you don't need to clone the repo to build an Observer). See [ContainerObserver](https://github.com/GitTorre/ContainerObserver) for a complete plugin impl that extends FO with SF container app resource monitoring and alerting. 4. Supports [Configuration Setting Application Updates](/Documentation/Using.md#parameterUpdates) for any observer for any supported setting.