From 240b355076557bd80eab92d8b664fb7223520f5f Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Tue, 12 Sep 2023 10:41:04 -0700 Subject: [PATCH 01/10] 3.2.11: Memory ops (Github version check). --- Build-SFPkgs.ps1 | 8 +++---- .../PackageRoot/Data/Plugins/Readme.txt | 2 +- .../Deployment/service-fabric-observer.json | 6 ++--- ...e-fabric-observer.v3.2.11.parameters.json} | 2 +- Documentation/OperationalTelemetry.md | 4 ++-- Documentation/Plugins.md | 2 +- Documentation/Using.md | 24 +++++++------------ FabricObserver.Extensibility.nuspec.template | 4 ++-- .../FabricObserver.Extensibility.csproj | 4 ++-- FabricObserver.nuspec.template | 9 +++---- FabricObserver.sln | 2 +- FabricObserver/FabricObserver.csproj | 4 ++-- FabricObserver/Observers/ObserverManager.cs | 13 ++++++---- .../PackageRoot/Data/Plugins/Readme.txt | 6 ++--- .../PackageRoot/ServiceManifest.xml | 8 +++---- .../PackageRoot/ServiceManifest_linux.xml | 8 +++---- .../ApplicationManifest.xml | 4 ++-- README.md | 12 +++++----- .../SampleObserverPlugin.csproj | 4 ++-- XmlDiffPatchSF/Program.cs | 2 +- foextlib.md | 2 +- fonuget.md | 2 +- 22 files changed, 63 insertions(+), 69 deletions(-) rename Documentation/Deployment/{service-fabric-observer.v3.2.10.parameters.json => service-fabric-observer.v3.2.11.parameters.json} (90%) diff --git a/Build-SFPkgs.ps1 b/Build-SFPkgs.ps1 index 29624bc3..7cdf5f16 100644 --- a/Build-SFPkgs.ps1 +++ b/Build-SFPkgs.ps1 @@ -23,11 +23,11 @@ function Build-SFPkg { try { Push-Location $scriptPath - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.SelfContained.3.2.10" "$scriptPath\bin\release\FabricObserver\linux-x64\self-contained\FabricObserverType" - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.FrameworkDependent.3.2.10" "$scriptPath\bin\release\FabricObserver\linux-x64\framework-dependent\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.SelfContained.3.2.11" "$scriptPath\bin\release\FabricObserver\linux-x64\self-contained\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Linux.FrameworkDependent.3.2.11" "$scriptPath\bin\release\FabricObserver\linux-x64\framework-dependent\FabricObserverType" - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.2.10" "$scriptPath\bin\release\FabricObserver\win-x64\self-contained\FabricObserverType" - Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.FrameworkDependent.3.2.10" "$scriptPath\bin\release\FabricObserver\win-x64\framework-dependent\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.2.11" "$scriptPath\bin\release\FabricObserver\win-x64\self-contained\FabricObserverType" + Build-SFPkg "Microsoft.ServiceFabricApps.FabricObserver.Windows.FrameworkDependent.3.2.11" "$scriptPath\bin\release\FabricObserver\win-x64\framework-dependent\FabricObserverType" } finally { Pop-Location diff --git a/ClusterObserver/PackageRoot/Data/Plugins/Readme.txt b/ClusterObserver/PackageRoot/Data/Plugins/Readme.txt index c7039f62..fe379207 100644 --- a/ClusterObserver/PackageRoot/Data/Plugins/Readme.txt +++ b/ClusterObserver/PackageRoot/Data/Plugins/Readme.txt @@ -68,5 +68,5 @@ cd C:\Users\me\source\repos\service-fabric-observer ./Build-FabricObserver ./Build-NugetPackages -The output from the above commands, FabricObserver platform-specific nupkgs and a package you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.10.nupkg, would be located in +The output from the above commands, FabricObserver platform-specific nupkgs and a package you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.11.nupkg, would be located in C:\Users\me\source\repos\service-fabric-observer\bin\release\FabricObserver\Nugets. \ No newline at end of file diff --git a/Documentation/Deployment/service-fabric-observer.json b/Documentation/Deployment/service-fabric-observer.json index 5673e375..83a36658 100644 --- a/Documentation/Deployment/service-fabric-observer.json +++ b/Documentation/Deployment/service-fabric-observer.json @@ -11,16 +11,16 @@ }, "applicationTypeVersionFabricObserver": { "type": "string", - "defaultValue": "3.2.10", + "defaultValue": "3.2.11", "metadata": { - "description": "Provide the app version number of FabricObserver. This must be identical to the version, 3.2.10, in the referenced sfpkg specified in packageUrlFabricObserver." + "description": "Provide the app version number of FabricObserver. This must be identical to the version, 3.2.11, in the referenced sfpkg specified in packageUrlFabricObserver." } }, "packageUrlFabricObserver": { "type": "string", "defaultValue": "", "metadata": { - "description": "This has to be a public accessible URL for the sfpkg file which contains the FabricObserver app package. Example: https://github.com/microsoft/service-fabric-observer/releases/download/[xxxxxxxx]/Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.2.10.sfpkg" + "description": "This has to be a public accessible URL for the sfpkg file which contains the FabricObserver app package. Example: https://github.com/microsoft/service-fabric-observer/releases/download/[xxxxxxxx]/Microsoft.ServiceFabricApps.FabricObserver.Windows.SelfContained.3.2.11.sfpkg" } } }, diff --git a/Documentation/Deployment/service-fabric-observer.v3.2.10.parameters.json b/Documentation/Deployment/service-fabric-observer.v3.2.11.parameters.json similarity index 90% rename from Documentation/Deployment/service-fabric-observer.v3.2.10.parameters.json rename to Documentation/Deployment/service-fabric-observer.v3.2.11.parameters.json index 5a0213a7..6e2a4060 100644 --- a/Documentation/Deployment/service-fabric-observer.v3.2.10.parameters.json +++ b/Documentation/Deployment/service-fabric-observer.v3.2.11.parameters.json @@ -6,7 +6,7 @@ "value": "" }, "applicationTypeVersionFabricObserver": { - "value": "3.2.10" + "value": "3.2.11" }, "packageUrlFabricObserver": { "value": "" diff --git a/Documentation/OperationalTelemetry.md b/Documentation/OperationalTelemetry.md index 3c8bf7d1..68b5a0b0 100644 --- a/Documentation/OperationalTelemetry.md +++ b/Documentation/OperationalTelemetry.md @@ -18,7 +18,7 @@ As with most of FabricObserver's application settings, you can also do this with Connect-ServiceFabricCluster ... $appParams = @{ "ObserverManagerEnableOperationalFOTelemetry" = "false"; } -Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationParameter $appParams -ApplicationTypeVersion 3.2.10 -UnMonitoredAuto +Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationParameter $appParams -ApplicationTypeVersion 3.2.11 -UnMonitoredAuto ``` @@ -44,7 +44,7 @@ Here is a full example of exactly what is sent in one of these telemetry events, "ClusterId": "00000000-1111-1111-0000-00f00d000d", "ClusterType": "SFRP", "NodeNameHash": "3e83569d4c6aad78083cd081215dafc81e5218556b6a46cb8dd2b183ed0095ad", - "FOVersion": "3.2.10", + "FOVersion": "3.2.11", "HasPlugins": "False", "SFRuntimeVersion":"9.0.1028.9590" "UpTime": "1.00:30:18.8058379", diff --git a/Documentation/Plugins.md b/Documentation/Plugins.md index 85633d6d..d5ce627a 100644 --- a/Documentation/Plugins.md +++ b/Documentation/Plugins.md @@ -72,5 +72,5 @@ cd C:\Users\me\source\repos\service-fabric-observer ./Build-FabricObserver ./Build-NugetPackages ``` -The output from the above commands contains FabricObserver platform-specific nupkgs and a nupkg you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.10.nupkg. Nuget packages will be located in +The output from the above commands contains FabricObserver platform-specific nupkgs and a nupkg you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.11.nupkg. Nuget packages will be located in C:\Users\me\source\repos\service-fabric-observer\bin\release\FabricObserver\Nugets. \ No newline at end of file diff --git a/Documentation/Using.md b/Documentation/Using.md index 44d5aa02..18303fe2 100644 --- a/Documentation/Using.md +++ b/Documentation/Using.md @@ -710,7 +710,7 @@ $appParams = @{ "FabricSystemObserverEnabled" = "true"; "FabricSystemObserverMem Then execute the application upgrade with ```Powershell -Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.2.10 -ApplicationParameter $appParams -Monitored -FailureAction rollback +Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.2.11 -ApplicationParameter $appParams -Monitored -FailureAction rollback ``` **Important**: This action will overwrite previous app paramemter changes that were made in an earlier application upgrade, for example. If you want to preserve any earlier changes, then you will need to @@ -718,10 +718,9 @@ supply those parameter values again along with the new ones. You do this in the ```PowerShell $appName = "fabric:/FabricObserver" -$appVersion = "3.2.10" - -$myApplication = Get-ServiceFabricApplication -ApplicationName $appName -$appParamCollection = $myApplication.ApplicationParameters +$appVersion = "3.2.11" +$application = Get-ServiceFabricApplication -ApplicationName $appName +$appParamCollection = $application.ApplicationParameters $applicationParameterMap = @{} # Fill the map with current app parameter settings. @@ -730,18 +729,13 @@ foreach ($pair in $appParamCollection) $applicationParameterMap.Add($pair.Name, $pair.Value); } -# If replacing current upgrade parameters (so, from a previous parameter-only application upgrade), remove them from the list of current params first. -if ($applicationParameterMap.ContainsKey("NodeObserverMemoryWarningLimitMb")) -{ - $applicationParameterMap.Remove("NodeObserverMemoryWarningLimitMb"); -} - -# Add the updated target app parameter(s) to the collection. -$applicationParameterMap.Add("NodeObserverMemoryWarningLimitMb","8000") +# Change existing app parameter values. +$applicationParameterMap["AppObserverEnableVerboseLogging"] = "true" -Start-ServiceFabricApplicationUpgrade -ApplicationName $appName -ApplicationTypeVersion $appVersion -ApplicationParameter $applicationParameterMap -Monitored -FailureAction Rollback +# Start the upgrade. +Start-ServiceFabricApplicationUpgrade -ApplicationName $appName -ApplicationTypeVersion $appVersion -ApplicationParameter $applicationParameterMap -UnMonitoredAuto ``` **For Linux, FO app parameter upgrades will restart FO processes** (one at a time, UD Walk with safety checks) due to the way Linux Capabilites work. In a nutshell, for any kind of application upgrade, we have to re-run the FO setup script to get the Capabilities in place, which requires restarting FabricObserver (which is just fine given that it is a stateless service). -**For Windows, FO processes will NOT be restarted as part of the upgrade UD walk**. +**For Windows, FO processes will NOT be restarted by default, but SF Hosting may decide to restart depending upon the number of parameters that are modified and/or the type of change (like changing RG values, for example)**. diff --git a/FabricObserver.Extensibility.nuspec.template b/FabricObserver.Extensibility.nuspec.template index abb72512..82a7320a 100644 --- a/FabricObserver.Extensibility.nuspec.template +++ b/FabricObserver.Extensibility.nuspec.template @@ -2,9 +2,9 @@ %PACKAGE_ID% - 3.2.10 + 3.2.11 -Note: This is library is required for observer plugins that target FabricObserver 3.2.10. +Note: This is library is required for observer plugins that target FabricObserver 3.2.11. Microsoft MIT diff --git a/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj b/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj index 7bdd2553..6fd3d3f4 100644 --- a/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj +++ b/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj @@ -5,8 +5,8 @@ FabricObserver Copyright © 2023 FabricObserver - 3.2.10 - 3.2.10 + 3.2.11 + 3.2.11 CA1416 diff --git a/FabricObserver.nuspec.template b/FabricObserver.nuspec.template index 1f83f27b..ac51c1ac 100644 --- a/FabricObserver.nuspec.template +++ b/FabricObserver.nuspec.template @@ -2,12 +2,9 @@ %PACKAGE_ID% - 3.2.10.2 + 3.2.11 -- This patch release fixes incorrect FO version in ApplicationManifest.xml that shipped in 3.2.10 nupkgs. -# 3.2.10 changes -- AppObserver Bug Fix: https://github.com/microsoft/service-fabric-observer/issues/276 -- Linux: Added Process object lifetime management in Bash() extension method. Removed hardcoded SF data root string ("/mnt/sfroot"). ServiceFabricConfiguration.Instance.FabricDataRoot is used instead so custom SF data root paths are supported. +- Memory usage improvement in Github release version check implementation. Microsoft MIT @@ -16,7 +13,7 @@ icon.png fonuget.md en-US - This package contains the FabricObserver(FO) Application - built for .NET 6.0 and SF Runtime 9.x. FO a highly configurable and extensible resource usage watchdog service that is designed to be run in Azure Service Fabric Windows and Linux clusters. This package contains the entire application and can be used to build .NET Standard 2.0 observer plugins. NOTE: If you want to target .NET 6 for your plugins, then you must use Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.10 nuget package to build them. + This package contains the FabricObserver(FO) Application - built for .NET 6.0 and SF Runtime 9.x. FO a highly configurable and extensible resource usage watchdog service that is designed to be run in Azure Service Fabric Windows and Linux clusters. This package contains the entire application and can be used to build .NET Standard 2.0 observer plugins. NOTE: If you want to target .NET 6 for your plugins, then you must use Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.11 nuget package to build them. diff --git a/FabricObserver.sln b/FabricObserver.sln index dcf08fe0..f5db8492 100644 --- a/FabricObserver.sln +++ b/FabricObserver.sln @@ -38,7 +38,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Documentation\Deployment\service-fabric-cluster-observer.v2.2.5.parameters.json = Documentation\Deployment\service-fabric-cluster-observer.v2.2.5.parameters.json Documentation\Deployment\service-fabric-observer.json = Documentation\Deployment\service-fabric-observer.json Documentation\Using.md = Documentation\Using.md - Documentation\Deployment\service-fabric-observer.v3.2.10.parameters.json = Documentation\Deployment\service-fabric-observer.v3.2.10.parameters.json + Documentation\Deployment\service-fabric-observer.v3.2.11.parameters.json = Documentation\Deployment\service-fabric-observer.v3.2.11.parameters.json EndProjectSection EndProject Project("{A07B5EB6-E848-4116-A8D0-A826331D98C6}") = "ClusterObserverApp", "ClusterObserverApp\ClusterObserverApp.sfproj", "{BD5D216F-5F89-4CC4-92FD-D6FDEC5A19AD}" diff --git a/FabricObserver/FabricObserver.csproj b/FabricObserver/FabricObserver.csproj index cb0a8c05..f33ff05e 100644 --- a/FabricObserver/FabricObserver.csproj +++ b/FabricObserver/FabricObserver.csproj @@ -11,8 +11,8 @@ True Copyright © 2022 FabricObserver - 3.2.10 - 3.2.10 + 3.2.11 + 3.2.11 true true FabricObserver.Program diff --git a/FabricObserver/Observers/ObserverManager.cs b/FabricObserver/Observers/ObserverManager.cs index 82730df2..de2019e3 100644 --- a/FabricObserver/Observers/ObserverManager.cs +++ b/FabricObserver/Observers/ObserverManager.cs @@ -53,7 +53,7 @@ private List Observers private CancellationTokenSource linkedSFRuntimeObserverTokenSource; // Folks often use their own version numbers. This is for internal diagnostic telemetry. - private const string InternalVersionNumber = "3.2.10"; + private const string InternalVersionNumber = "3.2.11"; private bool RuntimeTokenCancelled => linkedSFRuntimeObserverTokenSource?.Token.IsCancellationRequested ?? runAsyncToken.IsCancellationRequested; @@ -1286,15 +1286,15 @@ private async Task CheckGithubForNewVersionAsync() { try { - var githubClient = new GitHubClient(new ProductHeaderValue(ObserverConstants.FabricObserverName)); - IReadOnlyList releases = await githubClient.Repository.Release.GetAll("microsoft", "service-fabric-observer"); + GitHubClient githubClient = new(new ProductHeaderValue(ObserverConstants.FabricObserverName)); + Release latestRelease = await githubClient.Repository.Release.GetLatest("microsoft", "service-fabric-observer"); - if (releases.Count == 0) + if (latestRelease == null) { return; } - string releaseAssetName = releases[0].Name; + string releaseAssetName = latestRelease.Name; string latestVersion = releaseAssetName.Split(" ")[1]; Version latestGitHubVersion = new(latestVersion); Version localVersion = new(InternalVersionNumber); @@ -1344,6 +1344,9 @@ private async Task CheckGithubForNewVersionAsync() Logger.LogEtw(ObserverConstants.FabricObserverETWEventName, telemetryData); } } + + latestRelease = null; + githubClient = null; } catch (Exception e) when (e is not OutOfMemoryException) { diff --git a/FabricObserver/PackageRoot/Data/Plugins/Readme.txt b/FabricObserver/PackageRoot/Data/Plugins/Readme.txt index 76c8dc7d..42c8bfef 100644 --- a/FabricObserver/PackageRoot/Data/Plugins/Readme.txt +++ b/FabricObserver/PackageRoot/Data/Plugins/Readme.txt @@ -7,8 +7,8 @@ Note that the observer API lives in its own library, FabricObserver.Extensibilit 1. Create a new .NET 6 Library project. 2. Install the same version of the Microsoft.ServiceFabricApps.FabricObserver.Extensibility nupkg from https://www.nuget.org/profiles/ServiceFabricApps as the version of FabricObserver you are deploying. - E.g., 3.2.10 if you are going to deploy FO 3.2.10. - NOTE: You can also consume the entire FabricObserver 3.2.10 nupkg to build your plugin. Please see the SampleObserverPlugin project's csproj file for more information. + E.g., 3.2.11 if you are going to deploy FO 3.2.11. + NOTE: You can also consume the entire FabricObserver 3.2.11 nupkg to build your plugin. Please see the SampleObserverPlugin project's csproj file for more information. 3. Write an observer! @@ -68,5 +68,5 @@ cd C:\Users\me\source\repos\service-fabric-observer ./Build-FabricObserver ./Build-NugetPackages -The output from the above commands contains FabricObserver platform-specific nupkgs and a package you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.10.nupkg. Nupkg files from above command would be located in +The output from the above commands contains FabricObserver platform-specific nupkgs and a package you have to use for plugin authoring named Microsoft.ServiceFabricApps.FabricObserver.Extensibility.3.2.11.nupkg. Nupkg files from above command would be located in C:\Users\me\source\repos\service-fabric-observer\bin\release\FabricObserver\Nugets. \ No newline at end of file diff --git a/FabricObserver/PackageRoot/ServiceManifest.xml b/FabricObserver/PackageRoot/ServiceManifest.xml index 9cd50bb9..2356f394 100644 --- a/FabricObserver/PackageRoot/ServiceManifest.xml +++ b/FabricObserver/PackageRoot/ServiceManifest.xml @@ -1,6 +1,6 @@  @@ -9,7 +9,7 @@ This name must match the string used in RegisterServiceType call in Program.cs. --> - + install_lvid_perfcounter.bat @@ -25,10 +25,10 @@ - + - + \ No newline at end of file diff --git a/FabricObserver/PackageRoot/ServiceManifest_linux.xml b/FabricObserver/PackageRoot/ServiceManifest_linux.xml index 80a27ed5..d252968e 100644 --- a/FabricObserver/PackageRoot/ServiceManifest_linux.xml +++ b/FabricObserver/PackageRoot/ServiceManifest_linux.xml @@ -1,6 +1,6 @@  @@ -11,7 +11,7 @@ - + setcaps.sh @@ -27,10 +27,10 @@ - + - + \ No newline at end of file diff --git a/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml b/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml index c65cca5b..7a5f0700 100644 --- a/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml +++ b/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml @@ -1,6 +1,6 @@  - + @@ -236,7 +236,7 @@ should match the Name and Version attributes of the ServiceManifest element defined in the ServiceManifest.xml file. --> - + diff --git a/README.md b/README.md index 9cdeab06..cbef1d96 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## FabricObserver 3.2.10 +## FabricObserver 3.2.11 [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fmicrosoft%2Fservice-fabric-observer%2Fmain%2FDocumentation%2FDeployment%2Fservice-fabric-observer.json) @@ -87,7 +87,7 @@ see [FOAzurePipeline.yaml](/FOAzurePipeline.yaml) for msazure devops build tasks .net6 installed (if you deploy VM images from Azure gallery, then they will not have .net6 installed), then you must deploy the SelfContained package. ### Deploy FabricObserver -**Note: You must deploy this version (3.2.10) to clusters that are running SF 9.0 and above. This version also requires .NET 6.** +**Note: You must deploy this version (3.2.11) to clusters that are running SF 9.0 and above. This version also requires .NET 6.** You can deploy FabricObserver (and ClusterObserver) using Visual Studio (if you build the sources yourself), PowerShell or ARM. Please note that this version of FabricObserver no longer supports the DefaultServices node in ApplicationManifest.xml. This means that should you deploy using PowerShell, you must create an instance of the service as the last command in your script. This was done to support ARM deployment, specifically. The StartupServices.xml file you see in the FabricHealerApp project now contains the service information once held in ApplicationManifest's DefaultServices node. Note that this information is primarily useful for deploying from Visual Studio. @@ -127,15 +127,15 @@ Connect-ServiceFabricCluster -ConnectionEndpoint @('sf-win-cluster.westus2.cloud #Copy $path contents (FO app package) to server: -Copy-ServiceFabricApplicationPackage -ApplicationPackagePath $path -CompressPackage -ApplicationPackagePathInImageStore FO3210 -TimeoutSec 1800 +Copy-ServiceFabricApplicationPackage -ApplicationPackagePath $path -CompressPackage -ApplicationPackagePathInImageStore FO3211 -TimeoutSec 1800 #Register FO ApplicationType: -Register-ServiceFabricApplicationType -ApplicationPathInImageStore FO3210 +Register-ServiceFabricApplicationType -ApplicationPathInImageStore FO3211 #Create FO application (if not already deployed at lesser version): -New-ServiceFabricApplication -ApplicationName fabric:/FabricObserver -ApplicationTypeName FabricObserverType -ApplicationTypeVersion 3.2.10 +New-ServiceFabricApplication -ApplicationName fabric:/FabricObserver -ApplicationTypeName FabricObserverType -ApplicationTypeVersion 3.2.11 #Create the Service instances (-1 means all nodes, which is what is required for FO): @@ -143,7 +143,7 @@ New-ServiceFabricService -Stateless -PartitionSchemeSingleton -ApplicationName f #OR if updating existing version: -Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.2.10 -Monitored -FailureAction rollback +Start-ServiceFabricApplicationUpgrade -ApplicationName fabric:/FabricObserver -ApplicationTypeVersion 3.2.11 -Monitored -FailureAction rollback ``` ## Observer Model diff --git a/SampleObserverPlugin/SampleObserverPlugin.csproj b/SampleObserverPlugin/SampleObserverPlugin.csproj index 7ee789bd..6ea3828d 100644 --- a/SampleObserverPlugin/SampleObserverPlugin.csproj +++ b/SampleObserverPlugin/SampleObserverPlugin.csproj @@ -19,7 +19,7 @@ - - + diff --git a/XmlDiffPatchSF/Program.cs b/XmlDiffPatchSF/Program.cs index d5e2293e..870469ba 100644 --- a/XmlDiffPatchSF/Program.cs +++ b/XmlDiffPatchSF/Program.cs @@ -28,7 +28,7 @@ private static void Main(string[] args) "preceding the file extension.\n\n" + "**Note, if you have observer plugins, then you must supply true for [mergeExistingNodes] as the last argument to pull over your plugin settings as part of the merge.**.\n\n" + "Example:\n\n" + - "DiffPatchXml \"C:\\repos\\FO\\3.1.26\\configs\\ApplicationManifest.xml\" \"C:\\repos\\FO\\3.2.10\\configs\\ApplicationManifest.xml\"\n"); + "DiffPatchXml \"C:\\repos\\FO\\3.1.26\\configs\\ApplicationManifest.xml\" \"C:\\repos\\FO\\3.2.11\\configs\\ApplicationManifest.xml\"\n"); return; } diff --git a/foextlib.md b/foextlib.md index 456832da..f139fa47 100644 --- a/foextlib.md +++ b/foextlib.md @@ -1,4 +1,4 @@ -## FabricObserver Extensibility Library 3.2.10 +## FabricObserver Extensibility Library 3.2.11 FabricObserver.Extensibility is a .NET 6 library for building custom observers that extend FabricObserver's capabilities to match your needs. A custom observer is managed just like a built-in observer. diff --git a/fonuget.md b/fonuget.md index 95f53022..24a548ea 100644 --- a/fonuget.md +++ b/fonuget.md @@ -1,4 +1,4 @@ -## FabricObserver 3.2.10 +## FabricObserver 3.2.11 [**FabricObserver (FO)**](https://github.com/microsoft/service-fabric-observer) is a production-ready watchdog service with an easy-to-use extensibility model, written as a stateless, singleton Service Fabric **.NET 6** application that by default From f591eb766b327e968a4e03c49647a26bba86b669 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Tue, 12 Sep 2023 11:08:25 -0700 Subject: [PATCH 02/10] 3.2.11: Updated nupkgs (deps). --- .../FabricObserver.Extensibility.csproj | 2 +- FabricObserver/FabricObserver.csproj | 4 ++-- FabricObserverTests/FabricObserverTests.csproj | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj b/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj index 6fd3d3f4..660c5d75 100644 --- a/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj +++ b/FabricObserver.Extensibility/FabricObserver.Extensibility.csproj @@ -16,7 +16,7 @@ - + diff --git a/FabricObserver/FabricObserver.csproj b/FabricObserver/FabricObserver.csproj index f33ff05e..99a6e67e 100644 --- a/FabricObserver/FabricObserver.csproj +++ b/FabricObserver/FabricObserver.csproj @@ -30,8 +30,8 @@ - - + + diff --git a/FabricObserverTests/FabricObserverTests.csproj b/FabricObserverTests/FabricObserverTests.csproj index 847a223b..337f4f75 100644 --- a/FabricObserverTests/FabricObserverTests.csproj +++ b/FabricObserverTests/FabricObserverTests.csproj @@ -13,9 +13,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive From 8029d16b90115e0752b366d0a9f468d3e04b198e Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Tue, 19 Sep 2023 11:55:53 -0700 Subject: [PATCH 03/10] OSObserver WU check update. New test. --- FabricObserver.nuspec.template | 1 + FabricObserver/Observers/OSObserver.cs | 151 +++++++++--------- FabricObserver/Observers/ObserverManager.cs | 4 +- FabricObserverTests/ObserverTests.cs | 37 ++++- ...onfig.single-app-target-warning-ports.json | 2 +- 5 files changed, 115 insertions(+), 80 deletions(-) diff --git a/FabricObserver.nuspec.template b/FabricObserver.nuspec.template index ac51c1ac..1dd1bab1 100644 --- a/FabricObserver.nuspec.template +++ b/FabricObserver.nuspec.template @@ -5,6 +5,7 @@ 3.2.11 - Memory usage improvement in Github release version check implementation. +- Bug fix in OSObserver's Windows Update Automatic Download check app parameter update support. Microsoft MIT diff --git a/FabricObserver/Observers/OSObserver.cs b/FabricObserver/Observers/OSObserver.cs index a9f2d48f..e28c39ea 100644 --- a/FabricObserver/Observers/OSObserver.cs +++ b/FabricObserver/Observers/OSObserver.cs @@ -6,11 +6,13 @@ using System; using System.ComponentModel; using System.Fabric; +using System.Fabric.Description; using System.Fabric.Health; using System.Fabric.Query; using System.IO; using System.Linq; using System.Management; +using System.Runtime.InteropServices; using System.Security; using System.Text; using System.Threading; @@ -32,12 +34,9 @@ public sealed class OSObserver : ObserverBase private string osReport; private string osStatus; private bool auStateUnknown; - private bool isAUAutomaticDownloadEnabled; - private bool IsAUCheckSettingEnabled - { - get; set; - } + // For WU Automatic Download check. + public readonly bool IsWindowsDevCluster; public string ClusterManifestPath { @@ -50,7 +49,7 @@ public string ClusterManifestPath /// The StatelessServiceContext instance. public OSObserver(StatelessServiceContext context) : base(null, context) { - + IsWindowsDevCluster = IsWindowsDevClusterAsync().GetAwaiter().GetResult(); } public override async Task ObserveAsync(CancellationToken token) @@ -62,23 +61,10 @@ public override async Task ObserveAsync(CancellationToken token) } Token = token; - - // This only makes sense for Windows and only for non-dev clusters. - if (IsWindows) - { - await InitializeAUCheckAsync(); - - if (IsAUCheckSettingEnabled) - { - CheckWuAutoDownloadEnabled(); - } - } - await GetComputerInfoAsync(token); await ReportAsync(token); osReport = null; osStatus = null; - LastRunDateTime = DateTime.Now; } @@ -203,7 +189,7 @@ public override async Task ReportAsync(CancellationToken token) HealthReporter.ReportHealthToServiceFabric(report); // Windows Update automatic download enabled? - if (IsWindows && isAUAutomaticDownloadEnabled) + if (IsWindows && CheckWuAutoDownloadEnabled()) { CurrentWarningCount++; @@ -260,45 +246,73 @@ public override async Task ReportAsync(CancellationToken token) } } - private async Task InitializeAUCheckAsync() + private async Task IsOneNodeClusterAsync() { - if (!IsWindows) + try { - return; - } + var nodeQueryDesc = new NodeQueryDescription + { + MaxResults = 3, + }; - var checkAU = GetSettingParameterValue(ConfigurationSectionName, ObserverConstants.EnableWindowsAutoUpdateCheck); - bool isISDeployed = await IsInfrastructureServiceDeployedAsync(); + NodeList nodes = await FabricClientRetryHelper.ExecuteFabricActionWithRetryAsync( + () => FabricClientInstance.QueryManager.GetNodePagedListAsync( + nodeQueryDesc, + ConfigurationSettings.AsyncTimeout, + Token), + Token); - if (isISDeployed && !string.IsNullOrEmpty(checkAU) && bool.TryParse(checkAU, out bool auChk)) + return nodes != null && nodes.Count == 1; + } + catch (Exception e) when (e is FabricException or TaskCanceledException or TimeoutException) { - IsAUCheckSettingEnabled = auChk; + ObserverLogger.LogWarning($"IsOneNodeClusterAsync failure: {e.Message}"); + + // Don't know the answer, so false. + return false; } } - public async Task IsInfrastructureServiceDeployedAsync() + private async Task IsWindowsDevClusterAsync() { - try + if (IsWindowsDevCluster || await IsOneNodeClusterAsync()) { - var allSystemServices = - await FabricClientInstance.QueryManager.GetServiceListAsync( - new Uri(ObserverConstants.SystemAppName), - null, - AsyncClusterOperationTimeoutSeconds, Token); - - return allSystemServices.Count > 0 && allSystemServices.Any( - service => service.ServiceTypeName.Equals( - ObserverConstants.InfrastructureServiceType, - StringComparison.InvariantCultureIgnoreCase)); + return true; } - catch (Exception e) when (e is FabricException or TimeoutException) + + string clusterManifestXml = + !string.IsNullOrWhiteSpace(ClusterManifestPath) ? await File.ReadAllTextAsync(ClusterManifestPath, Token) + : await FabricClientInstance.ClusterManager.GetClusterManifestAsync(AsyncClusterOperationTimeoutSeconds, Token); + + // No-op. This dev cluster check only matters for Windows clusters (Windows auto-update check), so if for some reason we can't get the manifest, + // then assume dev cluster to block checking for AU download configuration. + if (string.IsNullOrWhiteSpace(clusterManifestXml)) + { + return true; + } + + if (clusterManifestXml.Contains("DevCluster", StringComparison.OrdinalIgnoreCase) + || ServiceFabricConfiguration.Instance.FabricDataRoot.Contains("DevCluster", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return false; + } + + private bool IsAutoUpdateDownloadCheckEnabled() + { + if (!IsWindows || IsWindowsDevCluster) { - ObserverLogger.LogWarning($"Unable to determine IS deployment status:{Environment.NewLine}{e}"); return false; } + + string checkAU = GetSettingParameterValue(ConfigurationSectionName, ObserverConstants.EnableWindowsAutoUpdateCheck); + + return !string.IsNullOrEmpty(checkAU) && bool.TryParse(checkAU, out bool auChk) && auChk; } - private static string GetWindowsHotFixes(bool generateKbUrl, CancellationToken token) + private string GetWindowsHotFixes(bool generateKbUrl, CancellationToken token) { if (!IsWindows) { @@ -356,28 +370,21 @@ private static string GetWindowsHotFixes(bool generateKbUrl, CancellationToken t sb = null; } - catch (Exception e) when ( - e is ArgumentException or - FormatException or - InvalidCastException or - ManagementException or - NullReferenceException) + catch (Exception e) when (e is not OutOfMemoryException) { - + ObserverLogger.LogWarning($"Unhandled Exception processing OS information: {e.Message}"); } return ret; } - private void CheckWuAutoDownloadEnabled() + private bool CheckWuAutoDownloadEnabled() { - if (!IsWindows) + if (!IsWindows || !IsAutoUpdateDownloadCheckEnabled()) { - return; + return false; } - Token.ThrowIfCancellationRequested(); - // Windows Update Automatic Download enabled (automatically downloading an update without notification beforehand)? // If so, it's best to disable this and deploy either POA (for Bronze durability clusters) // or enable VMSS automatic OS image upgrades for Silver+ durability clusters. @@ -385,19 +392,16 @@ private void CheckWuAutoDownloadEnabled() try { var wuLibAutoUpdates = new AutomaticUpdatesClass(); - isAUAutomaticDownloadEnabled = - wuLibAutoUpdates.ServiceEnabled && - wuLibAutoUpdates.Settings.NotificationLevel == AutomaticUpdatesNotificationLevel.aunlScheduledInstallation; + return wuLibAutoUpdates.ServiceEnabled && + wuLibAutoUpdates.Settings.NotificationLevel == AutomaticUpdatesNotificationLevel.aunlScheduledInstallation; } - catch (Exception e) when ( - e is System.Runtime.InteropServices.COMException or - InvalidOperationException or - SecurityException or - Win32Exception) + catch (Exception e) when (e is COMException or InvalidOperationException or SecurityException or Win32Exception) { - ObserverLogger.LogWarning($"{AuStateUnknownMessage}{Environment.NewLine}{e}"); + ObserverLogger.LogWarning($"{AuStateUnknownMessage}{Environment.NewLine}{e.Message}"); auStateUnknown = true; } + + return false; } private async Task GetComputerInfoAsync(CancellationToken token) @@ -419,13 +423,10 @@ private async Task GetComputerInfoAsync(CancellationToken token) string osEphemeralPortRange = string.Empty; string fabricAppPortRange = string.Empty; string clusterManifestXml = - !string.IsNullOrWhiteSpace(ClusterManifestPath) ? await File.ReadAllTextAsync(ClusterManifestPath, token) - : await FabricClientInstance.ClusterManager.GetClusterManifestAsync( - AsyncClusterOperationTimeoutSeconds, Token); - - (int lowPortApp, int highPortApp) = - NetworkUsage.TupleGetFabricApplicationPortRangeForNodeType(NodeType, clusterManifestXml); + !string.IsNullOrWhiteSpace(ClusterManifestPath) ? await File.ReadAllTextAsync(ClusterManifestPath, token) + : await FabricClientInstance.ClusterManager.GetClusterManifestAsync(AsyncClusterOperationTimeoutSeconds, Token); + (int lowPortApp, int highPortApp) = NetworkUsage.TupleGetFabricApplicationPortRangeForNodeType(NodeType, clusterManifestXml); int firewalls = NetworkUsage.GetActiveFirewallRulesCount(); // OS info. @@ -443,7 +444,7 @@ private async Task GetComputerInfoAsync(CancellationToken token) if (IsWindows) { // WU AutoUpdate - Automatic Download enabled. - if (IsAUCheckSettingEnabled) + if (IsAutoUpdateDownloadCheckEnabled()) { string auMessage = "WindowsUpdateAutoDownloadEnabled: "; @@ -453,7 +454,7 @@ private async Task GetComputerInfoAsync(CancellationToken token) } else { - auMessage += isAUAutomaticDownloadEnabled; + auMessage += CheckWuAutoDownloadEnabled(); } _ = sb.AppendLine(auMessage); } @@ -577,7 +578,7 @@ private async Task GetComputerInfoAsync(CancellationToken token) OSVersion = osInfo.Version, OSInstallDate = osInfo.InstallDate, LastBootUpTime = osInfo.LastBootUpTime, - WindowsUpdateAutoDownloadEnabled = isAUAutomaticDownloadEnabled, + WindowsUpdateAutoDownloadEnabled = CheckWuAutoDownloadEnabled(), TotalMemorySizeGB = (int)osInfo.TotalVisibleMemorySizeKB / 1048576, AvailablePhysicalMemoryGB = !IsWindows ? Math.Round(osInfo.AvailableMemoryKB / 1048576.0, 2) : Math.Round(osInfo.FreePhysicalMemoryKB / 1048576.0, 2), FreeVirtualMemoryGB = Math.Round(osInfo.FreeVirtualMemoryKB / 1048576.0, 2), @@ -608,9 +609,9 @@ private async Task GetComputerInfoAsync(CancellationToken token) _ = sb.Clear(); sb = null; } - catch (Exception e) when (e is not (OperationCanceledException or TaskCanceledException)) + catch (Exception e) when (e is not (OperationCanceledException or TaskCanceledException or OutOfMemoryException)) { - ObserverLogger.LogError($"Unhandled Exception processing OS information:{Environment.NewLine}{e}"); + ObserverLogger.LogError($"Unhandled Exception processing OS information: {e.Message}"); } } } diff --git a/FabricObserver/Observers/ObserverManager.cs b/FabricObserver/Observers/ObserverManager.cs index de2019e3..4dbd35e1 100644 --- a/FabricObserver/Observers/ObserverManager.cs +++ b/FabricObserver/Observers/ObserverManager.cs @@ -55,6 +55,8 @@ private List Observers // Folks often use their own version numbers. This is for internal diagnostic telemetry. private const string InternalVersionNumber = "3.2.11"; + private static FabricClient FabricClientInstance => FabricClientUtilities.FabricClientSingleton; + private bool RuntimeTokenCancelled => linkedSFRuntimeObserverTokenSource?.Token.IsCancellationRequested ?? runAsyncToken.IsCancellationRequested; @@ -108,8 +110,6 @@ public static StatelessServiceContext FabricServiceContext get; set; } - public static FabricClient FabricClientInstance => FabricClientUtilities.FabricClientSingleton; - public static bool TelemetryEnabled { get; set; diff --git a/FabricObserverTests/ObserverTests.cs b/FabricObserverTests/ObserverTests.cs index 57017cee..c98962c9 100644 --- a/FabricObserverTests/ObserverTests.cs +++ b/FabricObserverTests/ObserverTests.cs @@ -59,9 +59,10 @@ public static async Task TestClassStartUp(TestContext testContext) throw new Exception("Can't run these tests without a local dev cluster."); } + // TODO: Make this not the case.. if (!await IsOneNodeClusterAsync()) { - //throw new Exception("These tests need to be run in a 1-node SF dev cluster (one box) setup."); + throw new Exception("These tests need to be run in a 1-node SF dev cluster (one box) setup."); } // Remove orphaned health reports. @@ -129,7 +130,7 @@ public static async Task TestClassCleanupAsync() } await CleanupTestHealthReportsAsync(); - await RemoveTestApplicationsAsync(); + //await RemoveTestApplicationsAsync(); } /* Helpers */ @@ -2298,6 +2299,38 @@ public async Task OSObserver_ObserveAsync_Successful_IsHealthy_NoWarningsOrError Assert.IsTrue((await File.ReadAllLinesAsync(outputFilePath)).Length > 0); } + [TestMethod] + public async Task OSObserver_IsWindowsDevCluster_True() + { + var startDateTime = DateTime.Now; + + ObserverManager.FabricServiceContext = TestServiceContext; + ObserverManager.TelemetryEnabled = false; + ObserverManager.EtwEnabled = false; + + using var obs = new OSObserver(TestServiceContext) + { + ClusterManifestPath = Path.Combine(Environment.CurrentDirectory, "clusterManifest.xml"), + IsObserverWebApiAppDeployed = false, + IsEtwProviderEnabled = false + }; + + // This is required since output files are only created if fo api app is also deployed to cluster.. + + await obs.ObserveAsync(Token); + + // observer ran to completion with no errors. + Assert.IsTrue(obs.LastRunDateTime > startDateTime); + + // observer detected no error conditions. + Assert.IsFalse(obs.HasActiveFabricErrorOrWarning); + + // observer did not have any internal errors during run. + Assert.IsFalse(obs.IsUnhealthy); + + Assert.IsTrue(obs.IsWindowsDevCluster); + } + [TestMethod] public async Task DiskObserver_ObserveAsync_Successful_IsHealthy_NoWarningsOrErrors() { diff --git a/FabricObserverTests/PackageRoot/Config/AppObserver.config.single-app-target-warning-ports.json b/FabricObserverTests/PackageRoot/Config/AppObserver.config.single-app-target-warning-ports.json index 7fdeff2d..61dd34dc 100644 --- a/FabricObserverTests/PackageRoot/Config/AppObserver.config.single-app-target-warning-ports.json +++ b/FabricObserverTests/PackageRoot/Config/AppObserver.config.single-app-target-warning-ports.json @@ -3,7 +3,7 @@ "targetApp": "PortTest", "cpuWarningLimitPercent": 85, "memoryWarningLimitMb": 1024, - "networkWarningEphemeralPorts": 4, + "networkWarningEphemeralPorts": 1, "warningThreadCount": 500, "warningOpenFileHandles": 7000, "warningPrivateBytesMb": 1280, From 48145398e7b9cce31930d46f7d03accd150797f8 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Fri, 22 Sep 2023 15:35:18 -0700 Subject: [PATCH 04/10] rem process.Start conditional, updated FCUtilities. --- .../WindowsInfoProvider.cs | 15 +- .../ServiceFabric/FabricClientUtilities.cs | 253 +++++++++++++++++- 2 files changed, 255 insertions(+), 13 deletions(-) diff --git a/FabricObserver.Extensibility/Utilities/OperatingSystemInfo/WindowsInfoProvider.cs b/FabricObserver.Extensibility/Utilities/OperatingSystemInfo/WindowsInfoProvider.cs index fdb956cc..b146853b 100644 --- a/FabricObserver.Extensibility/Utilities/OperatingSystemInfo/WindowsInfoProvider.cs +++ b/FabricObserver.Extensibility/Utilities/OperatingSystemInfo/WindowsInfoProvider.cs @@ -251,11 +251,7 @@ public override (int LowPort, int HighPort, int NumberOfPorts) TupleGetDynamicPo }; process.StartInfo = ps; - - if (!process.Start()) - { - return (-1, -1, 0); - } + process.Start(); // Start async reads. process.BeginErrorReadLine(); @@ -276,7 +272,7 @@ public override (int LowPort, int HighPort, int NumberOfPorts) TupleGetDynamicPo { OSInfoLogger.LogWarning( "TupleGetDynamicPortRange: netsh failure. " + - $"Unable to determine dynamic port range (will return (-1, -1)):{Environment.NewLine}{error}"); + $"Unable to determine dynamic port range (will return (-1, -1)). Failed with: {error}"); return (-1, -1, 0); } @@ -645,12 +641,7 @@ private void RefreshNetstatData() }; process.StartInfo = ps; - - if (!process.Start()) - { - OSInfoLogger.LogWarning($"Unable to start process: {ps.Arguments}"); - return; - } + process.Start(); // Start asynchronous read operations. process.BeginErrorReadLine(); diff --git a/FabricObserver.Extensibility/Utilities/ServiceFabric/FabricClientUtilities.cs b/FabricObserver.Extensibility/Utilities/ServiceFabric/FabricClientUtilities.cs index 21961f55..e613066b 100644 --- a/FabricObserver.Extensibility/Utilities/ServiceFabric/FabricClientUtilities.cs +++ b/FabricObserver.Extensibility/Utilities/ServiceFabric/FabricClientUtilities.cs @@ -3,9 +3,11 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ +using FabricObserver.Observers.Interfaces; using FabricObserver.Observers.MachineInfoModel; using FabricObserver.Observers.Utilities; using FabricObserver.Observers.Utilities.Telemetry; +using Newtonsoft.Json.Linq; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -21,6 +23,7 @@ using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; +using System.Xml; using System.Xml.Serialization; using HealthReport = FabricObserver.Observers.Utilities.HealthReport; @@ -248,6 +251,246 @@ public async Task> GetAllDeployedReplicasO return repList; } + private void ProcessServiceConfiguration(string appTypeName, string codepackageName, ReplicaOrInstanceMonitoringInfo replicaInfo, CancellationToken token) + { + // ResourceGovernance/AppTypeVer/ServiceTypeVer. + logger.LogInfo($"Starting ProcessServiceConfiguration check for {replicaInfo.ServiceName.OriginalString}."); + + if (string.IsNullOrWhiteSpace(appTypeName)) + { + return; + } + + try + { + string appTypeVersion = null; + ApplicationParameterList appParameters = null; + ApplicationParameterList defaultParameters = null; + + ApplicationList appList = + FabricClientSingleton.QueryManager.GetApplicationListAsync( + replicaInfo.ApplicationName, + TimeSpan.FromSeconds(60), + token).Result; + + ApplicationTypeList applicationTypeList = + FabricClientSingleton.QueryManager.GetApplicationTypeListAsync( + appTypeName, + TimeSpan.FromSeconds(60), + token).Result; + + if (appList?.Count > 0) + { + try + { + if (appList.Any(app => app.ApplicationTypeName == appTypeName)) + { + appTypeVersion = appList.First(app => app.ApplicationTypeName == appTypeName).ApplicationTypeVersion; + appParameters = appList.First(app => app.ApplicationTypeName == appTypeName).ApplicationParameters; + replicaInfo.ApplicationTypeVersion = appTypeVersion; + } + + if (applicationTypeList.Any(app => app.ApplicationTypeVersion == appTypeVersion)) + { + defaultParameters = applicationTypeList.First(app => app.ApplicationTypeVersion == appTypeVersion).DefaultParameters; + } + } + catch (Exception e) when (e is ArgumentException or InvalidOperationException) + { + + } + + if (!string.IsNullOrWhiteSpace(appTypeVersion)) + { + // ServiceTypeVersion + var serviceList = + FabricClientSingleton.QueryManager.GetServiceListAsync( + replicaInfo.ApplicationName, + replicaInfo.ServiceName, + TimeSpan.FromSeconds(60), + token).Result; + + if (serviceList?.Count > 0) + { + try + { + Uri serviceName = replicaInfo.ServiceName; + + if (serviceList.Any(s => s.ServiceName == serviceName)) + { + replicaInfo.ServiceTypeVersion = serviceList.First(s => s.ServiceName == serviceName).ServiceManifestVersion; + } + } + catch (Exception e) when (e is ArgumentException or InvalidOperationException) + { + + } + } + } + } + } + catch (Exception e) when (e is AggregateException or FabricException or TaskCanceledException or TimeoutException or XmlException) + { + if (e is not TaskCanceledException) + { + logger.LogWarning($"Handled: Failed to process Service configuration for {replicaInfo.ServiceName.OriginalString} with exception '{e.Message}'"); + } + // move along + } + logger.LogInfo($"Completed ProcessServiceConfiguration for {replicaInfo.ServiceName.OriginalString}."); + } + + private void ProcessMultipleHelperCodePackages( + Uri appName, + string appTypeName, + DeployedServiceReplica deployedReplica, + ref ConcurrentQueue repsOrInstancesInfo, + bool isHostedByFabric, + bool enableChildProcessMonitoring, + NativeMethods.SafeObjectHandle handleToProcessSnapshot, + CancellationToken token) + { + logger.LogInfo($"Starting ProcessMultipleHelperCodePackages for {deployedReplica.ServiceName} (isHostedByFabric = {isHostedByFabric})"); + + if (repsOrInstancesInfo == null) + { + logger.LogInfo($"repsOrInstanceList is null. Exiting ProcessMultipleHelperCodePackages"); + return; + } + + try + { + DeployedCodePackageList codepackages = + FabricClientSingleton.QueryManager.GetDeployedCodePackageListAsync( + this.nodeName, + appName, + deployedReplica.ServiceManifestName, + null, + TimeSpan.FromSeconds(60), + token).Result; + + ReplicaOrInstanceMonitoringInfo replicaInfo = null; + + // Check for multiple code packages or GuestExecutable service (Fabric is the host). + if (codepackages.Count < 2 && !isHostedByFabric) + { + logger.LogInfo($"Completed ProcessMultipleHelperCodePackages."); + return; + } + + if (!codepackages.Any(c => c.CodePackageName != deployedReplica.CodePackageName)) + { + logger.LogInfo($"No helper code packages detected. Completed ProcessMultipleHelperCodePackages."); + return; + } + + var helperCodePackages = codepackages.Where(c => c.CodePackageName != deployedReplica.CodePackageName); + + foreach (var codepackage in helperCodePackages) + { + if (token.IsCancellationRequested) + { + return; + } + + int procId = (int)codepackage.EntryPoint.ProcessId; // The actual process id of the helper or guest executable binary. + string procName = null; + + if (OperatingSystem.IsWindows()) + { + try + { + procName = NativeMethods.GetProcessNameFromId(procId); + + if (procName == null) + { + continue; + } + } + catch (Win32Exception) + { + // Process no longer running or access denied. + continue; + } + } + else // Linux + { + using (var proc = Process.GetProcessById(procId)) + { + try + { + procName = proc.ProcessName; + } + catch (Exception e) when (e is InvalidOperationException or NotSupportedException or ArgumentException) + { + // Process no longer running. + continue; + } + } + } + + // This ensures that support for multiple CodePackages and GuestExecutable services fit naturally into AppObserver's *existing* implementation. + replicaInfo = new ReplicaOrInstanceMonitoringInfo + { + ApplicationName = appName, + ApplicationTypeName = appTypeName, + HostProcessId = procId, + HostProcessName = procName, + HostProcessStartTime = NativeMethods.GetProcessStartTime(procId), + ReplicaOrInstanceId = deployedReplica is DeployedStatefulServiceReplica replica ? + replica.ReplicaId : ((DeployedStatelessServiceInstance)deployedReplica).InstanceId, + PartitionId = deployedReplica.Partitionid, + ReplicaRole = deployedReplica is DeployedStatefulServiceReplica rep ? rep.ReplicaRole : ReplicaRole.None, + ServiceKind = deployedReplica.ServiceKind, + ServiceName = deployedReplica.ServiceName, + ServiceManifestName = codepackage.ServiceManifestName, + ServiceTypeName = deployedReplica.ServiceTypeName, + ServicePackageActivationId = string.IsNullOrWhiteSpace(codepackage.ServicePackageActivationId) ? + deployedReplica.ServicePackageActivationId : codepackage.ServicePackageActivationId, + ServicePackageActivationMode = string.IsNullOrWhiteSpace(codepackage.ServicePackageActivationId) ? + ServicePackageActivationMode.SharedProcess : ServicePackageActivationMode.ExclusiveProcess, + ReplicaStatus = deployedReplica is DeployedStatefulServiceReplica r ? + r.ReplicaStatus : ((DeployedStatelessServiceInstance)deployedReplica).ReplicaStatus, + }; + + // If Helper binaries launch child processes, AppObserver will monitor them, too. + if (enableChildProcessMonitoring && procId > 0) + { + // DEBUG - Perf +#if DEBUG + var sw = Stopwatch.StartNew(); +#endif + List<(string ProcName, int Pid, DateTime ProcessStartTime)> childPids = + ProcessInfoProvider.Instance.GetChildProcessInfo(procId, handleToProcessSnapshot); + + if (childPids != null && childPids.Count > 0) + { + replicaInfo.ChildProcesses = childPids; + logger.LogInfo($"{replicaInfo?.ServiceName}({procId}):{Environment.NewLine}Child procs (name, id, startDate): {string.Join(" ", replicaInfo.ChildProcesses)}"); + } +#if DEBUG + sw.Stop(); + logger.LogInfo($"EnableChildProcessMonitoring block run duration: {sw.Elapsed}"); +#endif + } + + // ResourceGovernance/AppTypeVer/ServiceTypeVer. + ProcessServiceConfiguration(appTypeName, codepackage.CodePackageName, replicaInfo, token); + + if (replicaInfo != null && replicaInfo.HostProcessId > 0 && + !repsOrInstancesInfo.Any(r => r.HostProcessId == replicaInfo.HostProcessId && r.HostProcessName == replicaInfo.HostProcessName)) + { + repsOrInstancesInfo.Enqueue(replicaInfo); + } + } + } + catch (Exception e) when (e is ArgumentException or FabricException or TaskCanceledException or TimeoutException) + { + logger.LogInfo($"ProcessMultipleHelperCodePackages: Handled Exception: {e.Message}"); + } + logger.LogInfo($"Completed ProcessMultipleHelperCodePackages."); + } + /// /// Returns a list of ReplicaOrInstanceMonitoringInfo objects that will contain child process information if handleToSnapshot is provided. /// @@ -367,7 +610,15 @@ any processes (children) that the service process (parent) created/spawned. */ } } - replicaMonitoringList.Enqueue(replicaInfo); + ProcessServiceConfiguration(applicationTypeName, deployedReplica.CodePackageName, replicaInfo, token); + + // null HostProcessName means the service process can't be monitored. If Fabric is the hosting process, then this is a Guest Executable or helper code package. + if (!string.IsNullOrWhiteSpace(replicaInfo.HostProcessName) && replicaInfo.HostProcessName != "Fabric") + { + replicaMonitoringList.Enqueue(replicaInfo); + } + + ProcessMultipleHelperCodePackages(appName, applicationTypeName, deployedReplica, ref replicaMonitoringList, replicaInfo.HostProcessName == "Fabric", includeChildProcesses, handleToSnapshot, token); } }); From e7fe35b468cfa8cf3004d18d1aca27296c0d2b00 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Sat, 23 Sep 2023 14:03:00 -0700 Subject: [PATCH 05/10] rem usings --- .../Utilities/OperatingSystemInfo/WindowsInfoProvider.cs | 4 ++-- .../Utilities/ProcessInfo/ProcessInfoProvider.cs | 1 - .../Utilities/ServiceFabric/FabricClientUtilities.cs | 2 -- FabricObserver/Observers/ObserverManager.cs | 6 +++--- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/FabricObserver.Extensibility/Utilities/OperatingSystemInfo/WindowsInfoProvider.cs b/FabricObserver.Extensibility/Utilities/OperatingSystemInfo/WindowsInfoProvider.cs index b146853b..8f97f23b 100644 --- a/FabricObserver.Extensibility/Utilities/OperatingSystemInfo/WindowsInfoProvider.cs +++ b/FabricObserver.Extensibility/Utilities/OperatingSystemInfo/WindowsInfoProvider.cs @@ -251,7 +251,7 @@ public override (int LowPort, int HighPort, int NumberOfPorts) TupleGetDynamicPo }; process.StartInfo = ps; - process.Start(); + _ = process.Start(); // Start async reads. process.BeginErrorReadLine(); @@ -641,7 +641,7 @@ private void RefreshNetstatData() }; process.StartInfo = ps; - process.Start(); + _ = process.Start(); // Start asynchronous read operations. process.BeginErrorReadLine(); diff --git a/FabricObserver.Extensibility/Utilities/ProcessInfo/ProcessInfoProvider.cs b/FabricObserver.Extensibility/Utilities/ProcessInfo/ProcessInfoProvider.cs index 61e22e73..c2cdb1eb 100644 --- a/FabricObserver.Extensibility/Utilities/ProcessInfo/ProcessInfoProvider.cs +++ b/FabricObserver.Extensibility/Utilities/ProcessInfo/ProcessInfoProvider.cs @@ -3,7 +3,6 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using NLog; using System; using System.Collections.Generic; using System.Diagnostics; diff --git a/FabricObserver.Extensibility/Utilities/ServiceFabric/FabricClientUtilities.cs b/FabricObserver.Extensibility/Utilities/ServiceFabric/FabricClientUtilities.cs index e613066b..994045ef 100644 --- a/FabricObserver.Extensibility/Utilities/ServiceFabric/FabricClientUtilities.cs +++ b/FabricObserver.Extensibility/Utilities/ServiceFabric/FabricClientUtilities.cs @@ -3,11 +3,9 @@ // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. // ------------------------------------------------------------ -using FabricObserver.Observers.Interfaces; using FabricObserver.Observers.MachineInfoModel; using FabricObserver.Observers.Utilities; using FabricObserver.Observers.Utilities.Telemetry; -using Newtonsoft.Json.Linq; using System; using System.Collections.Concurrent; using System.Collections.Generic; diff --git a/FabricObserver/Observers/ObserverManager.cs b/FabricObserver/Observers/ObserverManager.cs index 4dbd35e1..9eed658e 100644 --- a/FabricObserver/Observers/ObserverManager.cs +++ b/FabricObserver/Observers/ObserverManager.cs @@ -931,7 +931,7 @@ private void SetPropertiesFromConfigurationParameters(ConfigurationSettings sett IsLvidCounterEnabled = IsLVIDPerfCounterEnabled(settings); } - // ETW - Overridable + // ETW. if (bool.TryParse(GetConfigSettingValue(ObserverConstants.EnableETWProvider, settings), out bool etwEnabled)) { EtwEnabled = etwEnabled; @@ -1357,7 +1357,7 @@ private async Task CheckGithubForNewVersionAsync() private bool IsLVIDPerfCounterEnabled(ConfigurationSettings settings = null) { - if (!isWindows) + if (!isWindows /*|| ServiceFabricConfiguration.Instance.FabricVersion.StartsWith("10")*/) { return false; } @@ -1401,7 +1401,7 @@ private bool IsLVIDPerfCounterEnabled(ConfigurationSettings settings = null) } catch (Exception e) when (e is ArgumentException or InvalidOperationException or UnauthorizedAccessException or Win32Exception) { - Logger.LogWarning($"IsLVIDPerfCounterEnabled: Failed to determine LVID perf counter state:{Environment.NewLine}{e.Message}"); + Logger.LogWarning($"IsLVIDPerfCounterEnabled: Failed to determine LVID perf counter state: {e.Message}"); } return false; From 519e6e63c4a784390582a4fe82c6a122eaeac61c Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Sat, 23 Sep 2023 17:34:58 -0700 Subject: [PATCH 06/10] updated deps --- FabricObserver/FabricObserver.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FabricObserver/FabricObserver.csproj b/FabricObserver/FabricObserver.csproj index 99a6e67e..d5414c77 100644 --- a/FabricObserver/FabricObserver.csproj +++ b/FabricObserver/FabricObserver.csproj @@ -30,8 +30,8 @@ - - + + From 87a94454eb009fee8bf3be7395418d3d9aaccca9 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Mon, 25 Sep 2023 13:03:53 -0700 Subject: [PATCH 07/10] FO 3.2.11 --- FabricObserver.nuspec.template | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FabricObserver.nuspec.template b/FabricObserver.nuspec.template index 1dd1bab1..641ab166 100644 --- a/FabricObserver.nuspec.template +++ b/FabricObserver.nuspec.template @@ -4,8 +4,9 @@ %PACKAGE_ID% 3.2.11 +- Bug fix in OSObserver's Windows Update Automatic Download check. +- Bug fix in Windows dynamic port range detection. - Memory usage improvement in Github release version check implementation. -- Bug fix in OSObserver's Windows Update Automatic Download check app parameter update support. Microsoft MIT From 2de1aad97ad56b7d84d099a7c217fe40a794a77a Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Mon, 25 Sep 2023 14:50:19 -0700 Subject: [PATCH 08/10] FO 3.2.11 --- FabricObserver.nuspec.template | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FabricObserver.nuspec.template b/FabricObserver.nuspec.template index 641ab166..968f564a 100644 --- a/FabricObserver.nuspec.template +++ b/FabricObserver.nuspec.template @@ -5,8 +5,9 @@ 3.2.11 - Bug fix in OSObserver's Windows Update Automatic Download check. -- Bug fix in Windows dynamic port range detection. +- Bug fix in Windows dynamic port range detection (rare occurences, but still a bug). - Memory usage improvement in Github release version check implementation. +- Updated package dependencies. Microsoft MIT From 9d2fe771ca54706b5bc2eee31f401b1354ffbb49 Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Mon, 25 Sep 2023 14:58:51 -0700 Subject: [PATCH 09/10] FO 3.2.11 RTW --- FabricObserverTests/ObserverTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FabricObserverTests/ObserverTests.cs b/FabricObserverTests/ObserverTests.cs index c98962c9..305b683f 100644 --- a/FabricObserverTests/ObserverTests.cs +++ b/FabricObserverTests/ObserverTests.cs @@ -130,7 +130,7 @@ public static async Task TestClassCleanupAsync() } await CleanupTestHealthReportsAsync(); - //await RemoveTestApplicationsAsync(); + await RemoveTestApplicationsAsync(); } /* Helpers */ From d26fa1651739ba2869edf1c11d99868cc9b6a8db Mon Sep 17 00:00:00 2001 From: Charles Torre Date: Mon, 25 Sep 2023 15:05:32 -0700 Subject: [PATCH 10/10] 3.2.11 - default settings. --- .../ApplicationPackageRoot/ApplicationManifest.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml b/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml index 7a5f0700..ea47e0a5 100644 --- a/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml +++ b/FabricObserverApp/ApplicationPackageRoot/ApplicationManifest.xml @@ -91,11 +91,11 @@ - - + + - +