From 129afab0397c717023ff1ec842d8c70c4756f039 Mon Sep 17 00:00:00 2001 From: JT Date: Sun, 16 Jun 2024 21:09:35 -0700 Subject: [PATCH 01/25] Re-add the Rename functionality for checkpoints --- .../CheckpointFileViewModel.cs | 73 +++++++++++++++++++ .../ViewModels/CheckpointsPageViewModel.cs | 1 + .../Views/CheckpointsPage.axaml | 4 +- 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/StabilityMatrix.Avalonia/ViewModels/CheckpointManager/CheckpointFileViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/CheckpointManager/CheckpointFileViewModel.cs index 97095bc2..0264b9ca 100644 --- a/StabilityMatrix.Avalonia/ViewModels/CheckpointManager/CheckpointFileViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/CheckpointManager/CheckpointFileViewModel.cs @@ -1,11 +1,14 @@ using System; using System.Collections.Immutable; using System.ComponentModel; +using System.IO; using System.Threading.Tasks; using Avalonia.Controls.Notifications; +using Avalonia.Data; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; using FluentAvalonia.UI.Controls; +using Microsoft.Extensions.Logging; using StabilityMatrix.Avalonia.Services; using StabilityMatrix.Avalonia.ViewModels.Base; using StabilityMatrix.Avalonia.ViewModels.Dialogs; @@ -36,6 +39,7 @@ public partial class CheckpointFileViewModel : SelectableViewModelBase private readonly IModelIndexService modelIndexService; private readonly INotificationService notificationService; private readonly ServiceManager vmFactory; + private readonly ILogger logger; public bool CanShowTriggerWords => CheckpointFile.ConnectedModelInfo?.TrainedWords?.Length > 0; public string BaseModelName => CheckpointFile.ConnectedModelInfo?.BaseModel ?? string.Empty; @@ -47,6 +51,7 @@ public CheckpointFileViewModel( IModelIndexService modelIndexService, INotificationService notificationService, ServiceManager vmFactory, + ILogger logger, LocalModelFile checkpointFile ) { @@ -54,6 +59,7 @@ LocalModelFile checkpointFile this.modelIndexService = modelIndexService; this.notificationService = notificationService; this.vmFactory = vmFactory; + this.logger = logger; CheckpointFile = checkpointFile; ThumbnailUri = settingsManager.IsLibraryDirSet ? CheckpointFile.GetPreviewImageFullPath(settingsManager.ModelsDirectory) @@ -186,4 +192,71 @@ private async Task DeleteAsync(bool showConfirmation = true) await modelIndexService.RemoveModelAsync(CheckpointFile); } + + [RelayCommand] + private async Task RenameAsync() + { + // Parent folder path + var parentPath = + Path.GetDirectoryName((string?)CheckpointFile.GetFullPath(settingsManager.ModelsDirectory)) ?? ""; + + var textFields = new TextBoxField[] + { + new() + { + Label = "File name", + Validator = text => + { + if (string.IsNullOrWhiteSpace(text)) + throw new DataValidationException("File name is required"); + + if (File.Exists(Path.Combine(parentPath, text))) + throw new DataValidationException("File name already exists"); + }, + Text = CheckpointFile.FileName + } + }; + + var dialog = DialogHelper.CreateTextEntryDialog("Rename Model", "", textFields); + + if (await dialog.ShowAsync() == ContentDialogResult.Primary) + { + var name = textFields[0].Text; + var nameNoExt = Path.GetFileNameWithoutExtension(name); + var originalNameNoExt = Path.GetFileNameWithoutExtension(CheckpointFile.FileName); + // Rename file in OS + try + { + var newFilePath = Path.Combine(parentPath, name); + File.Move(CheckpointFile.GetFullPath(settingsManager.ModelsDirectory), newFilePath); + + // If preview image exists, rename it too + var previewPath = CheckpointFile.GetPreviewImageFullPath(settingsManager.ModelsDirectory); + if (previewPath != null && File.Exists(previewPath)) + { + var newPreviewImagePath = Path.Combine( + parentPath, + $"{nameNoExt}.preview{Path.GetExtension(previewPath)}" + ); + File.Move(previewPath, newPreviewImagePath); + } + + // If connected model info exists, rename it too (.cm-info.json) + if (CheckpointFile.HasConnectedModel) + { + var cmInfoPath = Path.Combine(parentPath, $"{originalNameNoExt}.cm-info.json"); + if (File.Exists(cmInfoPath)) + { + File.Move(cmInfoPath, Path.Combine(parentPath, $"{nameNoExt}.cm-info.json")); + } + } + + await modelIndexService.RefreshIndex(); + } + catch (Exception e) + { + logger.LogError(e, "Failed to rename checkpoint file"); + } + } + } } diff --git a/StabilityMatrix.Avalonia/ViewModels/CheckpointsPageViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/CheckpointsPageViewModel.cs index a1aebc6e..8ddac7b5 100644 --- a/StabilityMatrix.Avalonia/ViewModels/CheckpointsPageViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/CheckpointsPageViewModel.cs @@ -274,6 +274,7 @@ or nameof(SortConnectedModelsFirst) modelIndexService, notificationService, dialogFactory, + logger, x ) ) diff --git a/StabilityMatrix.Avalonia/Views/CheckpointsPage.axaml b/StabilityMatrix.Avalonia/Views/CheckpointsPage.axaml index f6eb440e..86b7a0f2 100644 --- a/StabilityMatrix.Avalonia/Views/CheckpointsPage.axaml +++ b/StabilityMatrix.Avalonia/Views/CheckpointsPage.axaml @@ -367,8 +367,8 @@ IsVisible="{Binding CheckpointFile.HasConnectedModel}" /> - - + Date: Sun, 16 Jun 2024 21:10:44 -0700 Subject: [PATCH 02/25] chagenlog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e992dd7b..5ae5cfb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to Stability Matrix will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html). +## v2.11.1 +### Added +- Added Rename option back to the Checkpoints page + ## v2.11.0 ### Added #### Packages From 9fdf750cd3b05cf29a26d3d3982334e7c49ad872 Mon Sep 17 00:00:00 2001 From: JT Date: Sun, 16 Jun 2024 21:17:24 -0700 Subject: [PATCH 03/25] fix build --- StabilityMatrix.Avalonia/DesignData/DesignData.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/StabilityMatrix.Avalonia/DesignData/DesignData.cs b/StabilityMatrix.Avalonia/DesignData/DesignData.cs index 89b561bb..d2cbce9c 100644 --- a/StabilityMatrix.Avalonia/DesignData/DesignData.cs +++ b/StabilityMatrix.Avalonia/DesignData/DesignData.cs @@ -327,6 +327,7 @@ public static void Initialize() new MockModelIndexService(), notificationService, dialogFactory, + null, new LocalModelFile { SharedFolderType = SharedFolderType.StableDiffusion, @@ -356,6 +357,7 @@ public static void Initialize() new MockModelIndexService(), notificationService, dialogFactory, + null, new LocalModelFile { RelativePath = "~/Models/Lora/model.safetensors", From dcadcb35dc5354ff0134fd4262ee266d397c3291 Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 14:04:59 -0400 Subject: [PATCH 04/25] Add pinned numpy and mpmath for comfy install --- StabilityMatrix.Core/Models/Packages/ComfyUI.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/StabilityMatrix.Core/Models/Packages/ComfyUI.cs b/StabilityMatrix.Core/Models/Packages/ComfyUI.cs index 8fa0f3c4..04effa62 100644 --- a/StabilityMatrix.Core/Models/Packages/ComfyUI.cs +++ b/StabilityMatrix.Core/Models/Packages/ComfyUI.cs @@ -231,12 +231,7 @@ public override async Task InstallPackage( ) }; - switch (torchVersion) - { - case TorchVersion.Mps: - pipArgs = pipArgs.AddArg("mpmath==1.3.0"); - break; - } + pipArgs = pipArgs.AddArg("numpy==1.26.4").AddArg("mpmath==1.3.0"); var requirements = new FilePath(installLocation, "requirements.txt"); From 1a7200cc9447c237abc75c2ef38c99fc7bbe55d4 Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 14:13:09 -0400 Subject: [PATCH 05/25] Add changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ae5cfb6..02ab6df0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2 ## v2.11.1 ### Added - Added Rename option back to the Checkpoints page +### Fixed +- Fixed [#689](https://github.com/LykosAI/StabilityMatrix/issues/689) - New ComfyUI installs encountering launch error due to torch 2.0.0 update, added pinned `numpy==1.26.4` to install and update. ## v2.11.0 ### Added From 9f53880e72b0c435debded881513fc90492ce47f Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 16:30:35 -0400 Subject: [PATCH 06/25] Add stale workflow --- .github/workflows/stale.yml | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..9e7038d5 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,39 @@ +name: 'Close stale issues' + +permissions: + issues: write + pull-requests: write + +on: + workflow_dispatch: + schedule: + - cron: '30 1 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: 'This issue is pending because it has been awaiting a response for 14 days with no activity. Remove the pending label or comment, else this will be closed in 5 days.' + close-issue-message: 'This issue was closed because it has been pending for 5 days with no activity.' + only-labels: 'awaiting-feedback' + stale-issue-label: 'pending' + exempt-issue-labels: 'planned,milestone,work-in-progress' + days-before-issue-stale: 14 + days-before-issue-close: 5 + days-before-pr-close: -1 + days-before-pr-stale: -1 + operations-per-run: 45 + + - uses: actions/stale@v9 + with: + stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove the stale label or comment, else this will be closed in 5 days.' + close-issue-message: 'This issue was closed because it has been stale for 5 days with no activity.' + stale-issue-label: 'stale' + exempt-issue-labels: 'planned,milestone,work-in-progress' + days-before-issue-stale: 30 + days-before-issue-close: 5 + days-before-pr-close: -1 + days-before-pr-stale: -1 + operations-per-run: 45 From 8147de41aa029e025241b8943f6a595edb3f3846 Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 16:30:57 -0400 Subject: [PATCH 07/25] Formatting --- StabilityMatrix.Avalonia/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/StabilityMatrix.Avalonia/Program.cs b/StabilityMatrix.Avalonia/Program.cs index 558914fb..2f148887 100644 --- a/StabilityMatrix.Avalonia/Program.cs +++ b/StabilityMatrix.Avalonia/Program.cs @@ -349,7 +349,7 @@ UnobservedTaskExceptionEventArgs e { if (e.Exception is Exception ex) { - Logger.Error(ex, "Unobserved task exception"); + Logger.Error(ex, "Unobserved Task Exception"); } } From 643777e6820f25e83751dfcfadf8ae7a63ab74d3 Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 16:33:50 -0400 Subject: [PATCH 08/25] Add unobserved task exception notification --- CHANGELOG.md | 2 ++ StabilityMatrix.Avalonia/App.axaml.cs | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02ab6df0..ba982a18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2 ## v2.11.1 ### Added - Added Rename option back to the Checkpoints page +### Changed +- Unobserved Task Exceptions across the app will now show a toast notification to aid in debugging ### Fixed - Fixed [#689](https://github.com/LykosAI/StabilityMatrix/issues/689) - New ComfyUI installs encountering launch error due to torch 2.0.0 update, added pinned `numpy==1.26.4` to install and update. diff --git a/StabilityMatrix.Avalonia/App.axaml.cs b/StabilityMatrix.Avalonia/App.axaml.cs index 88c38002..1e5c9b9e 100644 --- a/StabilityMatrix.Avalonia/App.axaml.cs +++ b/StabilityMatrix.Avalonia/App.axaml.cs @@ -294,6 +294,7 @@ private void ShowMainWindow() DesktopLifetime.ShutdownRequested += OnShutdownRequested; AppDomain.CurrentDomain.ProcessExit += OnExit; + TaskScheduler.UnobservedTaskException += TaskScheduler_UnobservedTaskException; // Since we're manually shutting down NLog in OnExit LogManager.AutoShutdown = false; @@ -883,6 +884,32 @@ private void OnExit(object? sender, EventArgs _) } } + private static void TaskScheduler_UnobservedTaskException( + object? sender, + UnobservedTaskExceptionEventArgs e + ) + { + if (e.Exception is not Exception unobservedEx) + return; + + try + { + var notificationService = Services.GetRequiredService(); + + notificationService.ShowPersistent( + $"Unobserved Task Exception - {unobservedEx.GetType().Name}", + unobservedEx.Message + ); + + // Consider the exception observed if we were able to show a notification + e.SetObserved(); + } + catch (Exception ex) + { + Logger.Error(ex, "Failed to show Unobserved Task Exception notification"); + } + } + private static LoggingConfiguration ConfigureLogging() { var setupBuilder = LogManager.Setup(); From a0a0d83ffee26f53eb0600d103ebb22d63a36d9f Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 16:36:02 -0400 Subject: [PATCH 09/25] Fix paint canvas load bitmap not working --- CHANGELOG.md | 1 + .../ViewModels/Controls/PaintCanvasViewModel.cs | 10 ++++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba982a18..57445aa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2 - Unobserved Task Exceptions across the app will now show a toast notification to aid in debugging ### Fixed - Fixed [#689](https://github.com/LykosAI/StabilityMatrix/issues/689) - New ComfyUI installs encountering launch error due to torch 2.0.0 update, added pinned `numpy==1.26.4` to install and update. +- Fixed Inference image mask editor's 'Load Mask' not able to load image files ## v2.11.0 ### Added diff --git a/StabilityMatrix.Avalonia/ViewModels/Controls/PaintCanvasViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Controls/PaintCanvasViewModel.cs index fed45ceb..b94afe7c 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Controls/PaintCanvasViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Controls/PaintCanvasViewModel.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.Collections.Immutable; using System.ComponentModel; using System.Linq; @@ -77,6 +76,9 @@ public partial class PaintCanvasViewModel(ILogger logger) [JsonIgnore] private SKLayer BrushLayer => Layers["Brush"]; + [JsonIgnore] + private SKLayer ImagesLayer => Layers["Images"]; + [JsonIgnore] private SKLayer BackgroundLayer => Layers["Background"]; @@ -99,9 +101,6 @@ public SKBitmap? BackgroundImage } } - [JsonIgnore] - public List LayerImages { get; } = []; - /// /// Set by to allow the view model to /// refresh the canvas view after updating points or bitmap layers. @@ -117,8 +116,7 @@ public void SetSourceCanvas(SKCanvas canvas) public void LoadCanvasFromBitmap(SKBitmap bitmap) { - LayerImages.Clear(); - LayerImages.Add(bitmap); + ImagesLayer.Bitmaps = [bitmap]; RefreshCanvas?.Invoke(); } From 1dc62e3b8dbe16d4f4c61d067f049314f2c82e2a Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 20:19:39 -0400 Subject: [PATCH 10/25] Fix unobserved task exception notification thread --- StabilityMatrix.Avalonia/App.axaml.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/StabilityMatrix.Avalonia/App.axaml.cs b/StabilityMatrix.Avalonia/App.axaml.cs index 1e5c9b9e..54e65754 100644 --- a/StabilityMatrix.Avalonia/App.axaml.cs +++ b/StabilityMatrix.Avalonia/App.axaml.cs @@ -896,10 +896,14 @@ UnobservedTaskExceptionEventArgs e { var notificationService = Services.GetRequiredService(); - notificationService.ShowPersistent( - $"Unobserved Task Exception - {unobservedEx.GetType().Name}", - unobservedEx.Message - ); + Dispatcher.UIThread.Invoke(() => + { + var originException = unobservedEx.InnerException ?? unobservedEx; + notificationService.ShowPersistent( + $"Unobserved Task Exception - {originException.GetType().Name}", + originException.Message + ); + }); // Consider the exception observed if we were able to show a notification e.SetObserved(); From 71fddebbfd729e84f547e9c64106e04f0eacae1c Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 20:20:07 -0400 Subject: [PATCH 11/25] Add SharedFolderLayout framework --- .../Helper/SharedFoldersConfigHelper.cs | 207 ++++++++++++++++++ .../Packages/ISharedFolderLayoutPackage.cs | 41 ++++ .../Models/Packages/SharedFolderLayout.cs | 42 ++++ .../Models/Packages/SharedFolderLayoutRule.cs | 40 ++++ 4 files changed, 330 insertions(+) create mode 100644 StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs create mode 100644 StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs create mode 100644 StabilityMatrix.Core/Models/Packages/SharedFolderLayout.cs create mode 100644 StabilityMatrix.Core/Models/Packages/SharedFolderLayoutRule.cs diff --git a/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs b/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs new file mode 100644 index 00000000..8ec3ab13 --- /dev/null +++ b/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs @@ -0,0 +1,207 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using StabilityMatrix.Core.Extensions; +using StabilityMatrix.Core.Models.Packages; + +namespace StabilityMatrix.Core.Helper; + +public static class SharedFoldersConfigHelper +{ + /// + /// Updates a JSON object with shared folder layout rules, using the SourceTypes, + /// converted to absolute paths using the sharedModelsDirectory. + /// + public static void UpdateJsonConfigForShared( + SharedFolderLayout layout, + JsonObject jsonObject, + string sharedModelsDirectory, + SharedFoldersConfigOptions? options = null + ) + { + UpdateJsonConfig( + layout, + jsonObject, + rule => + rule.SourceTypes.Select(type => Path.Combine(sharedModelsDirectory, type.GetStringValue())), + options + ); + } + + public static async Task UpdateJsonConfigFileForSharedAsync( + SharedFolderLayout layout, + Stream configStream, + string sharedModelsDirectory, + SharedFoldersConfigOptions? options = null + ) + { + options ??= SharedFoldersConfigOptions.Default; + + JsonObject jsonNode; + + if (configStream.Length == 0) + { + jsonNode = new JsonObject(); + } + else + { + jsonNode = + await JsonSerializer + .DeserializeAsync(configStream, options.JsonSerializerOptions) + .ConfigureAwait(false) ?? new JsonObject(); + } + + UpdateJsonConfigForShared(layout, jsonNode, sharedModelsDirectory, options); + + configStream.Seek(0, SeekOrigin.Begin); + configStream.SetLength(0); + + await JsonSerializer + .SerializeAsync(configStream, jsonNode, options.JsonSerializerOptions) + .ConfigureAwait(false); + } + + public static async Task UpdateJsonConfigFileForSharedAsync( + SharedFolderLayout layout, + string packageRootDirectory, + string sharedModelsDirectory, + SharedFoldersConfigOptions? options = null + ) + { + var configPath = Path.Combine( + packageRootDirectory, + layout.RelativeConfigPath ?? throw new InvalidOperationException("RelativeConfigPath is null") + ); + + await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); + + await UpdateJsonConfigFileForSharedAsync(layout, stream, sharedModelsDirectory, options) + .ConfigureAwait(false); + } + + /// + /// Updates a JSON object with shared folder layout rules, using the TargetRelativePaths, + /// converted to absolute paths using the packageRootDirectory. + /// + public static void UpdateJsonConfigForDefault( + SharedFolderLayout layout, + JsonObject jsonObject, + string packageRootDirectory, + SharedFoldersConfigOptions? options = null + ) + { + UpdateJsonConfig( + layout, + jsonObject, + rule => + rule.TargetRelativePaths.Select(NormalizePathSlashes) + .Select(path => Path.Combine(packageRootDirectory, path)), + options + ); + } + + public static async Task UpdateJsonConfigFileForDefaultAsync( + SharedFolderLayout layout, + Stream configStream, + string packageRootDirectory, + SharedFoldersConfigOptions? options = null + ) + { + options ??= SharedFoldersConfigOptions.Default; + + JsonObject jsonNode; + + if (configStream.Length == 0) + { + jsonNode = new JsonObject(); + } + else + { + jsonNode = + await JsonSerializer + .DeserializeAsync(configStream, options.JsonSerializerOptions) + .ConfigureAwait(false) ?? new JsonObject(); + } + + UpdateJsonConfigForDefault(layout, jsonNode, packageRootDirectory, options); + + configStream.Seek(0, SeekOrigin.Begin); + configStream.SetLength(0); + + await JsonSerializer + .SerializeAsync(configStream, jsonNode, options.JsonSerializerOptions) + .ConfigureAwait(false); + } + + public static async Task UpdateJsonConfigFileForDefaultAsync( + SharedFolderLayout layout, + string packageRootDirectory, + SharedFoldersConfigOptions? options = null + ) + { + var configPath = Path.Combine( + packageRootDirectory, + layout.RelativeConfigPath ?? throw new InvalidOperationException("RelativeConfigPath is null") + ); + + await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); + + await UpdateJsonConfigFileForDefaultAsync(layout, stream, packageRootDirectory, options) + .ConfigureAwait(false); + } + + public static void UpdateJsonConfig( + SharedFolderLayout layout, + JsonObject jsonObject, + Func> pathsSelector, + SharedFoldersConfigOptions? options = null + ) + { + options ??= SharedFoldersConfigOptions.Default; + + var rulesByConfigPath = layout.GetRulesByConfigPath(); + + foreach (var (configPath, rule) in rulesByConfigPath) + { + // Get paths to write with selector + var paths = pathsSelector(rule).ToArray(); + + // Multiple elements or alwaysWriteArray is true, write as array + if (paths.Length > 1 || options.AlwaysWriteArray) + { + jsonObject[configPath] = new JsonArray( + paths.Select(path => (JsonNode)JsonValue.Create(path)).ToArray() + ); + } + // 1 element and alwaysWriteArray is false, write as string + else if (paths.Length == 1) + { + jsonObject[configPath] = paths[0]; + } + else + { + jsonObject.Remove(configPath); + } + } + } + + private static string NormalizePathSlashes(string path) + { + if (Compat.IsWindows) + { + return path.Replace('/', '\\'); + } + + return path; + } + + public class SharedFoldersConfigOptions + { + public bool AlwaysWriteArray { get; set; } = false; + + public static SharedFoldersConfigOptions Default => new(); + + public JsonSerializerOptions JsonSerializerOptions = + new() { WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault }; + } +} diff --git a/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs b/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs new file mode 100644 index 00000000..48f19b09 --- /dev/null +++ b/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs @@ -0,0 +1,41 @@ +using System.Collections.Immutable; + +namespace StabilityMatrix.Core.Models.Packages; + +public interface ISharedFolderLayoutPackage +{ + SharedFolderLayout SharedFolderLayout { get; } + + Dictionary> LegacySharedFolders + { + get + { + var result = new Dictionary>(); + + foreach (var rule in SharedFolderLayout.Rules) + { + if (rule.TargetRelativePaths is not { Length: > 0 } value) + { + continue; + } + + foreach (var folderTypeKey in rule.SourceTypes) + { + var existingList = + (ImmutableList) + result.GetValueOrDefault(folderTypeKey, ImmutableList.Empty); + + foreach (var path in value) + { + if (!existingList.Contains(path)) + { + result[folderTypeKey] = existingList.Add(path); + } + } + } + } + + return result; + } + } +} diff --git a/StabilityMatrix.Core/Models/Packages/SharedFolderLayout.cs b/StabilityMatrix.Core/Models/Packages/SharedFolderLayout.cs new file mode 100644 index 00000000..2b121668 --- /dev/null +++ b/StabilityMatrix.Core/Models/Packages/SharedFolderLayout.cs @@ -0,0 +1,42 @@ +using System.Collections.Immutable; + +namespace StabilityMatrix.Core.Models.Packages; + +public record SharedFolderLayout +{ + /// + /// Optional config file path, relative from package installation directory + /// + public string? RelativeConfigPath { get; set; } + + public IImmutableList Rules { get; set; } = []; + + public Dictionary GetRulesByConfigPath() + { + // Dictionary of config path to rule + var configPathToRule = new Dictionary(); + + foreach (var rule in Rules) + { + // Ignore rules without config paths + if (rule.ConfigDocumentPaths is not { Length: > 0 } configPaths) + { + continue; + } + + foreach (var configPath in configPaths) + { + // Get or create rule + var existingRule = configPathToRule.GetValueOrDefault( + configPath, + new SharedFolderLayoutRule() + ); + + // Add unique + configPathToRule[configPath] = existingRule.Union(rule); + } + } + + return configPathToRule; + } +} diff --git a/StabilityMatrix.Core/Models/Packages/SharedFolderLayoutRule.cs b/StabilityMatrix.Core/Models/Packages/SharedFolderLayoutRule.cs new file mode 100644 index 00000000..8b795bc2 --- /dev/null +++ b/StabilityMatrix.Core/Models/Packages/SharedFolderLayoutRule.cs @@ -0,0 +1,40 @@ +namespace StabilityMatrix.Core.Models.Packages; + +public readonly record struct SharedFolderLayoutRule +{ + public SharedFolderType[] SourceTypes { get; init; } + + public string[] TargetRelativePaths { get; init; } + + public string[] ConfigDocumentPaths { get; init; } + + public SharedFolderLayoutRule() + { + SourceTypes = []; + TargetRelativePaths = []; + ConfigDocumentPaths = []; + } + + public SharedFolderLayoutRule(SharedFolderType[] types, string[] targets) + { + SourceTypes = types; + TargetRelativePaths = targets; + } + + public SharedFolderLayoutRule(SharedFolderType[] types, string[] targets, string[] configs) + { + SourceTypes = types; + TargetRelativePaths = targets; + ConfigDocumentPaths = configs; + } + + public SharedFolderLayoutRule Union(SharedFolderLayoutRule other) + { + return this with + { + SourceTypes = SourceTypes.Union(other.SourceTypes).ToArray(), + TargetRelativePaths = TargetRelativePaths.Union(other.TargetRelativePaths).ToArray(), + ConfigDocumentPaths = ConfigDocumentPaths.Union(other.ConfigDocumentPaths).ToArray() + }; + } +} From 9a0e6b2346a1adc7064e8e7ab10c3434818a5175 Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 20:22:14 -0400 Subject: [PATCH 12/25] Update Fooocus to use SharedFolderLayout --- .../Models/Packages/Fooocus.cs | 201 +++++++++--------- 1 file changed, 100 insertions(+), 101 deletions(-) diff --git a/StabilityMatrix.Core/Models/Packages/Fooocus.cs b/StabilityMatrix.Core/Models/Packages/Fooocus.cs index 3899fa0e..134f2d06 100644 --- a/StabilityMatrix.Core/Models/Packages/Fooocus.cs +++ b/StabilityMatrix.Core/Models/Packages/Fooocus.cs @@ -1,7 +1,4 @@ using System.Diagnostics; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.Json.Serialization; using System.Text.RegularExpressions; using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Helper; @@ -21,7 +18,9 @@ public class Fooocus( ISettingsManager settingsManager, IDownloadService downloadService, IPrerequisiteHelper prerequisiteHelper -) : BaseGitPackage(githubApi, settingsManager, downloadService, prerequisiteHelper) +) + : BaseGitPackage(githubApi, settingsManager, downloadService, prerequisiteHelper), + ISharedFolderLayoutPackage { public override string Name => "Fooocus"; public override string DisplayName { get; set; } = "Fooocus"; @@ -154,22 +153,95 @@ IPrerequisiteHelper prerequisiteHelper new[] { SharedFolderMethod.Symlink, SharedFolderMethod.Configuration, SharedFolderMethod.None }; public override Dictionary> SharedFolders => + ((ISharedFolderLayoutPackage)this).LegacySharedFolders; + + public virtual SharedFolderLayout SharedFolderLayout => new() { - [SharedFolderType.StableDiffusion] = new[] { "models/checkpoints" }, - [SharedFolderType.Diffusers] = new[] { "models/diffusers" }, - [SharedFolderType.Lora] = new[] { "models/loras" }, - [SharedFolderType.CLIP] = new[] { "models/clip" }, - [SharedFolderType.TextualInversion] = new[] { "models/embeddings" }, - [SharedFolderType.VAE] = new[] { "models/vae" }, - [SharedFolderType.ApproxVAE] = new[] { "models/vae_approx" }, - [SharedFolderType.ControlNet] = new[] { "models/controlnet" }, - [SharedFolderType.GLIGEN] = new[] { "models/gligen" }, - [SharedFolderType.ESRGAN] = new[] { "models/upscale_models" }, - [SharedFolderType.Hypernetwork] = new[] { "models/hypernetworks" } + RelativeConfigPath = "config.txt", + Rules = + [ + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.StableDiffusion], + TargetRelativePaths = ["models/checkpoints"], + ConfigDocumentPaths = ["path_checkpoints"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.Diffusers], + TargetRelativePaths = ["models/diffusers"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.CLIP], + TargetRelativePaths = ["models/clip"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.GLIGEN], + TargetRelativePaths = ["models/gligen"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.ESRGAN], + TargetRelativePaths = ["models/upscale_models"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.Hypernetwork], + TargetRelativePaths = ["models/hypernetworks"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.TextualInversion], + TargetRelativePaths = ["models/embeddings"], + ConfigDocumentPaths = ["path_embeddings"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.VAE], + TargetRelativePaths = ["models/vae"], + ConfigDocumentPaths = ["path_vae"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.ApproxVAE], + TargetRelativePaths = ["models/vae_approx"], + ConfigDocumentPaths = ["path_vae_approx"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.Lora, SharedFolderType.LyCORIS], + TargetRelativePaths = ["models/loras"], + ConfigDocumentPaths = ["path_loras"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.InvokeClipVision], + TargetRelativePaths = ["models/clip_vision"], + ConfigDocumentPaths = ["path_clip_vision"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.ControlNet], + TargetRelativePaths = ["models/controlnet"], + ConfigDocumentPaths = ["path_controlnet"] + }, + new SharedFolderLayoutRule + { + TargetRelativePaths = ["models/inpaint"], + ConfigDocumentPaths = ["path_inpaint"] + }, + new SharedFolderLayoutRule + { + TargetRelativePaths = ["models/prompt_expansion/fooocus_expansion"], + ConfigDocumentPaths = ["path_fooocus_expansion"] + } + ] }; - public override Dictionary>? SharedOutputFolders => + public override Dictionary> SharedOutputFolders => new() { [SharedOutputType.Text2Img] = new[] { "outputs" } }; public override IEnumerable AvailableTorchVersions => @@ -193,7 +265,7 @@ public override async Task InstallPackage( ) { var venvRunner = await SetupVenv(installLocation, forceRecreate: true).ConfigureAwait(false); - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + venvRunner.EnvironmentVariables = SettingsManager.Settings.EnvironmentVariables; progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); @@ -274,7 +346,12 @@ SharedFolderMethod sharedFolderMethod { SharedFolderMethod.Symlink => base.SetupModelFolders(installDirectory, SharedFolderMethod.Symlink), - SharedFolderMethod.Configuration => SetupModelFoldersConfig(installDirectory), + SharedFolderMethod.Configuration + => SharedFoldersConfigHelper.UpdateJsonConfigFileForSharedAsync( + SharedFolderLayout, + installDirectory, + SettingsManager.ModelsDirectory + ), SharedFolderMethod.None => Task.CompletedTask, _ => throw new ArgumentOutOfRangeException(nameof(sharedFolderMethod), sharedFolderMethod, null) }; @@ -288,91 +365,13 @@ SharedFolderMethod sharedFolderMethod return sharedFolderMethod switch { SharedFolderMethod.Symlink => base.RemoveModelFolderLinks(installDirectory, sharedFolderMethod), - SharedFolderMethod.Configuration => WriteDefaultConfig(installDirectory), + SharedFolderMethod.Configuration + => SharedFoldersConfigHelper.UpdateJsonConfigFileForDefaultAsync( + SharedFolderLayout, + installDirectory + ), SharedFolderMethod.None => Task.CompletedTask, _ => throw new ArgumentOutOfRangeException(nameof(sharedFolderMethod), sharedFolderMethod, null) }; } - - private JsonSerializerOptions jsonSerializerOptions = - new() { WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault }; - - private async Task SetupModelFoldersConfig(DirectoryPath installDirectory) - { - var fooocusConfigPath = installDirectory.JoinFile("config.txt"); - - var fooocusConfig = new JsonObject(); - - if (fooocusConfigPath.Exists) - { - fooocusConfig = - JsonSerializer.Deserialize( - await fooocusConfigPath.ReadAllTextAsync().ConfigureAwait(false) - ) ?? new JsonObject(); - } - - fooocusConfig["path_checkpoints"] = Path.Combine(settingsManager.ModelsDirectory, "StableDiffusion"); - fooocusConfig["path_loras"] = new JsonArray - { - Path.Combine(settingsManager.ModelsDirectory, "Lora"), - Path.Combine(settingsManager.ModelsDirectory, "LyCORIS") - }; - fooocusConfig["path_embeddings"] = Path.Combine(settingsManager.ModelsDirectory, "TextualInversion"); - fooocusConfig["path_vae_approx"] = Path.Combine(settingsManager.ModelsDirectory, "ApproxVAE"); - fooocusConfig["path_upscale_models"] = Path.Combine(settingsManager.ModelsDirectory, "ESRGAN"); - fooocusConfig["path_inpaint"] = Path.Combine(installDirectory, "models", "inpaint"); - fooocusConfig["path_controlnet"] = Path.Combine(settingsManager.ModelsDirectory, "ControlNet"); - fooocusConfig["path_clip_vision"] = Path.Combine(settingsManager.ModelsDirectory, "CLIP"); - fooocusConfig["path_fooocus_expansion"] = Path.Combine( - installDirectory, - "models", - "prompt_expansion", - "fooocus_expansion" - ); - - var outputsPath = Path.Combine(installDirectory, OutputFolderName); - - // doesn't always exist on first install - Directory.CreateDirectory(outputsPath); - fooocusConfig["path_outputs"] = outputsPath; - - await fooocusConfigPath - .WriteAllTextAsync(JsonSerializer.Serialize(fooocusConfig, jsonSerializerOptions)) - .ConfigureAwait(false); - } - - private async Task WriteDefaultConfig(DirectoryPath installDirectory) - { - var fooocusConfigPath = installDirectory.JoinFile("config.txt"); - - var fooocusConfig = new JsonObject(); - - if (fooocusConfigPath.Exists) - { - fooocusConfig = - JsonSerializer.Deserialize( - await fooocusConfigPath.ReadAllTextAsync().ConfigureAwait(false) - ) ?? new JsonObject(); - } - - fooocusConfig["path_checkpoints"] = Path.Combine(installDirectory, "models", "checkpoints"); - fooocusConfig["path_loras"] = Path.Combine(installDirectory, "models", "loras"); - fooocusConfig["path_embeddings"] = Path.Combine(installDirectory, "models", "embeddings"); - fooocusConfig["path_vae_approx"] = Path.Combine(installDirectory, "models", "vae_approx"); - fooocusConfig["path_upscale_models"] = Path.Combine(installDirectory, "models", "upscale_models"); - fooocusConfig["path_inpaint"] = Path.Combine(installDirectory, "models", "inpaint"); - fooocusConfig["path_controlnet"] = Path.Combine(installDirectory, "models", "controlnet"); - fooocusConfig["path_clip_vision"] = Path.Combine(installDirectory, "models", "clip_vision"); - fooocusConfig["path_fooocus_expansion"] = Path.Combine( - installDirectory, - "models", - "prompt_expansion", - "fooocus_expansion" - ); - fooocusConfig["path_outputs"] = Path.Combine(installDirectory, OutputFolderName); - - await fooocusConfigPath - .WriteAllTextAsync(JsonSerializer.Serialize(fooocusConfig, jsonSerializerOptions)) - .ConfigureAwait(false); - } } From 68ef26b1cbd42e2879a34083ebce735d8293b82f Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 20:28:07 -0400 Subject: [PATCH 13/25] Add output folder sharing --- .../Models/Packages/Fooocus.cs | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/StabilityMatrix.Core/Models/Packages/Fooocus.cs b/StabilityMatrix.Core/Models/Packages/Fooocus.cs index 134f2d06..93a717f1 100644 --- a/StabilityMatrix.Core/Models/Packages/Fooocus.cs +++ b/StabilityMatrix.Core/Models/Packages/Fooocus.cs @@ -237,6 +237,11 @@ IPrerequisiteHelper prerequisiteHelper { TargetRelativePaths = ["models/prompt_expansion/fooocus_expansion"], ConfigDocumentPaths = ["path_fooocus_expansion"] + }, + new SharedFolderLayoutRule + { + TargetRelativePaths = [OutputFolderName], + ConfigDocumentPaths = ["path_outputs"] } ] }; @@ -346,12 +351,7 @@ SharedFolderMethod sharedFolderMethod { SharedFolderMethod.Symlink => base.SetupModelFolders(installDirectory, SharedFolderMethod.Symlink), - SharedFolderMethod.Configuration - => SharedFoldersConfigHelper.UpdateJsonConfigFileForSharedAsync( - SharedFolderLayout, - installDirectory, - SettingsManager.ModelsDirectory - ), + SharedFolderMethod.Configuration => SetupModelFoldersConfig(installDirectory), SharedFolderMethod.None => Task.CompletedTask, _ => throw new ArgumentOutOfRangeException(nameof(sharedFolderMethod), sharedFolderMethod, null) }; @@ -365,13 +365,30 @@ SharedFolderMethod sharedFolderMethod return sharedFolderMethod switch { SharedFolderMethod.Symlink => base.RemoveModelFolderLinks(installDirectory, sharedFolderMethod), - SharedFolderMethod.Configuration - => SharedFoldersConfigHelper.UpdateJsonConfigFileForDefaultAsync( - SharedFolderLayout, - installDirectory - ), + SharedFolderMethod.Configuration => WriteDefaultConfig(installDirectory), SharedFolderMethod.None => Task.CompletedTask, _ => throw new ArgumentOutOfRangeException(nameof(sharedFolderMethod), sharedFolderMethod, null) }; } + + private async Task SetupModelFoldersConfig(DirectoryPath installDirectory) + { + // doesn't always exist on first install + installDirectory.JoinDir(OutputFolderName).Create(); + + await SharedFoldersConfigHelper + .UpdateJsonConfigFileForDefaultAsync(SharedFolderLayout, installDirectory) + .ConfigureAwait(false); + } + + private Task WriteDefaultConfig(DirectoryPath installDirectory) + { + // doesn't always exist on first install + installDirectory.JoinDir(OutputFolderName).Create(); + + return SharedFoldersConfigHelper.UpdateJsonConfigFileForDefaultAsync( + SharedFolderLayout, + installDirectory + ); + } } From 31ad97b27fb3b531be40f0886a61f113638afa7f Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 20:35:41 -0400 Subject: [PATCH 14/25] Add unique behavior for LegacySharedFolders conversion --- .../Packages/ISharedFolderLayoutPackage.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs b/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs index 48f19b09..58d1e9a0 100644 --- a/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs +++ b/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs @@ -10,6 +10,9 @@ Dictionary> LegacySharedFolders { get { + // Keep track of unique paths since symbolic links can't do multiple targets + // So we'll ignore duplicates once they appear here + var addedPaths = new HashSet(); var result = new Dictionary>(); foreach (var rule in SharedFolderLayout.Rules) @@ -27,10 +30,15 @@ Dictionary> LegacySharedFolders foreach (var path in value) { - if (!existingList.Contains(path)) - { - result[folderTypeKey] = existingList.Add(path); - } + // Skip if the path is already in the list + if (existingList.Contains(path)) + continue; + + // Skip if the path is already added globally + if (!addedPaths.Add(path)) + continue; + + result[folderTypeKey] = existingList.Add(path); } } } From 090a203092acdfcb495580fc563512fa776b4766 Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 20:36:36 -0400 Subject: [PATCH 15/25] Fix fooocus controlnet config shared folders not working --- CHANGELOG.md | 1 + .../Models/Packages/FocusControlNet.cs | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57445aa9..0852c04d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2 ### Fixed - Fixed [#689](https://github.com/LykosAI/StabilityMatrix/issues/689) - New ComfyUI installs encountering launch error due to torch 2.0.0 update, added pinned `numpy==1.26.4` to install and update. - Fixed Inference image mask editor's 'Load Mask' not able to load image files +- Fixed Fooocus ControlNet default config shared folder mode not taking effect ## v2.11.0 ### Added diff --git a/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs b/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs index 9291c1e8..f8fa9a19 100644 --- a/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs +++ b/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs @@ -1,12 +1,6 @@ -using System.Diagnostics; -using System.Text.RegularExpressions; -using StabilityMatrix.Core.Attributes; +using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; -using StabilityMatrix.Core.Models.FileInterfaces; -using StabilityMatrix.Core.Models.Progress; -using StabilityMatrix.Core.Processes; -using StabilityMatrix.Core.Python; using StabilityMatrix.Core.Services; namespace StabilityMatrix.Core.Models.Packages; @@ -23,9 +17,16 @@ IPrerequisiteHelper prerequisiteHelper public override string DisplayName { get; set; } = "Fooocus-ControlNet"; public override string Author => "fenneishi"; public override string Blurb => "Fooocus-ControlNet adds more control to the original Fooocus software."; - public override string LicenseUrl => "https://github.com/fenneishi/Fooocus-ControlNet-SDXL/blob/main/LICENSE"; + public override string LicenseUrl => + "https://github.com/fenneishi/Fooocus-ControlNet-SDXL/blob/main/LICENSE"; public override Uri PreviewImageUri => new("https://github.com/fenneishi/Fooocus-ControlNet-SDXL/raw/main/asset/canny/snip.png"); public override PackageDifficulty InstallerSortOrder => PackageDifficulty.Expert; public override bool OfferInOneClickInstaller => false; + + public override SharedFolderLayout SharedFolderLayout => + base.SharedFolderLayout with + { + RelativeConfigPath = "user_path_config.txt" + }; } From 1bb6ce5de026655d71b2421cd521effa31e0bbc1 Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 20:40:02 -0400 Subject: [PATCH 16/25] Version bump --- StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj b/StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj index 63993011..255e9a37 100644 --- a/StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj +++ b/StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj @@ -17,7 +17,7 @@ app.manifest true ./Assets/Icon.ico - 2.11.0-dev.999 + 2.12.0-dev.999 $(Version) true true From 81eca9151203340de6816cd212410e0cc6037c2e Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 21:00:08 -0400 Subject: [PATCH 17/25] Move options to own file --- .../Helper/SharedFoldersConfigHelper.cs | 11 ----------- .../Helper/SharedFoldersConfigOptions.cs | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 StabilityMatrix.Core/Helper/SharedFoldersConfigOptions.cs diff --git a/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs b/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs index 8ec3ab13..49a4d6d1 100644 --- a/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs +++ b/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs @@ -1,6 +1,5 @@ using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.Json.Serialization; using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Models.Packages; @@ -194,14 +193,4 @@ private static string NormalizePathSlashes(string path) return path; } - - public class SharedFoldersConfigOptions - { - public bool AlwaysWriteArray { get; set; } = false; - - public static SharedFoldersConfigOptions Default => new(); - - public JsonSerializerOptions JsonSerializerOptions = - new() { WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault }; - } } diff --git a/StabilityMatrix.Core/Helper/SharedFoldersConfigOptions.cs b/StabilityMatrix.Core/Helper/SharedFoldersConfigOptions.cs new file mode 100644 index 00000000..5cdb5048 --- /dev/null +++ b/StabilityMatrix.Core/Helper/SharedFoldersConfigOptions.cs @@ -0,0 +1,14 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StabilityMatrix.Core.Helper; + +public class SharedFoldersConfigOptions +{ + public static SharedFoldersConfigOptions Default => new(); + + public bool AlwaysWriteArray { get; set; } = false; + + public JsonSerializerOptions JsonSerializerOptions { get; set; } = + new() { WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault }; +} From b96a770d9343af9e191105d9e6ffb4cafe00c154 Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 21:06:56 -0400 Subject: [PATCH 18/25] Hide the overloads --- .../Helper/SharedFoldersConfigHelper.cs | 94 +++++++++---------- .../Models/Packages/Fooocus.cs | 6 +- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs b/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs index 49a4d6d1..f6c7ea22 100644 --- a/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs +++ b/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs @@ -7,27 +7,25 @@ namespace StabilityMatrix.Core.Helper; public static class SharedFoldersConfigHelper { - /// - /// Updates a JSON object with shared folder layout rules, using the SourceTypes, - /// converted to absolute paths using the sharedModelsDirectory. - /// - public static void UpdateJsonConfigForShared( + public static async Task UpdateJsonConfigFileForSharedAsync( SharedFolderLayout layout, - JsonObject jsonObject, + string packageRootDirectory, string sharedModelsDirectory, SharedFoldersConfigOptions? options = null ) { - UpdateJsonConfig( - layout, - jsonObject, - rule => - rule.SourceTypes.Select(type => Path.Combine(sharedModelsDirectory, type.GetStringValue())), - options + var configPath = Path.Combine( + packageRootDirectory, + layout.RelativeConfigPath ?? throw new InvalidOperationException("RelativeConfigPath is null") ); + + await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); + + await UpdateJsonConfigFileForSharedAsync(layout, stream, sharedModelsDirectory, options) + .ConfigureAwait(false); } - public static async Task UpdateJsonConfigFileForSharedAsync( + private static async Task UpdateJsonConfigFileForSharedAsync( SharedFolderLayout layout, Stream configStream, string sharedModelsDirectory, @@ -60,32 +58,14 @@ await JsonSerializer .ConfigureAwait(false); } - public static async Task UpdateJsonConfigFileForSharedAsync( - SharedFolderLayout layout, - string packageRootDirectory, - string sharedModelsDirectory, - SharedFoldersConfigOptions? options = null - ) - { - var configPath = Path.Combine( - packageRootDirectory, - layout.RelativeConfigPath ?? throw new InvalidOperationException("RelativeConfigPath is null") - ); - - await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); - - await UpdateJsonConfigFileForSharedAsync(layout, stream, sharedModelsDirectory, options) - .ConfigureAwait(false); - } - /// - /// Updates a JSON object with shared folder layout rules, using the TargetRelativePaths, - /// converted to absolute paths using the packageRootDirectory. + /// Updates a JSON object with shared folder layout rules, using the SourceTypes, + /// converted to absolute paths using the sharedModelsDirectory. /// - public static void UpdateJsonConfigForDefault( + private static void UpdateJsonConfigForShared( SharedFolderLayout layout, JsonObject jsonObject, - string packageRootDirectory, + string sharedModelsDirectory, SharedFoldersConfigOptions? options = null ) { @@ -93,13 +73,29 @@ public static void UpdateJsonConfigForDefault( layout, jsonObject, rule => - rule.TargetRelativePaths.Select(NormalizePathSlashes) - .Select(path => Path.Combine(packageRootDirectory, path)), + rule.SourceTypes.Select(type => Path.Combine(sharedModelsDirectory, type.GetStringValue())), options ); } public static async Task UpdateJsonConfigFileForDefaultAsync( + SharedFolderLayout layout, + string packageRootDirectory, + SharedFoldersConfigOptions? options = null + ) + { + var configPath = Path.Combine( + packageRootDirectory, + layout.RelativeConfigPath ?? throw new InvalidOperationException("RelativeConfigPath is null") + ); + + await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); + + await UpdateJsonConfigFileForDefaultAsync(layout, stream, packageRootDirectory, options) + .ConfigureAwait(false); + } + + private static async Task UpdateJsonConfigFileForDefaultAsync( SharedFolderLayout layout, Stream configStream, string packageRootDirectory, @@ -132,24 +128,28 @@ await JsonSerializer .ConfigureAwait(false); } - public static async Task UpdateJsonConfigFileForDefaultAsync( + /// + /// Updates a JSON object with shared folder layout rules, using the TargetRelativePaths, + /// converted to absolute paths using the packageRootDirectory. + /// + private static void UpdateJsonConfigForDefault( SharedFolderLayout layout, + JsonObject jsonObject, string packageRootDirectory, SharedFoldersConfigOptions? options = null ) { - var configPath = Path.Combine( - packageRootDirectory, - layout.RelativeConfigPath ?? throw new InvalidOperationException("RelativeConfigPath is null") + UpdateJsonConfig( + layout, + jsonObject, + rule => + rule.TargetRelativePaths.Select(NormalizePathSlashes) + .Select(path => Path.Combine(packageRootDirectory, path)), + options ); - - await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); - - await UpdateJsonConfigFileForDefaultAsync(layout, stream, packageRootDirectory, options) - .ConfigureAwait(false); } - public static void UpdateJsonConfig( + private static void UpdateJsonConfig( SharedFolderLayout layout, JsonObject jsonObject, Func> pathsSelector, diff --git a/StabilityMatrix.Core/Models/Packages/Fooocus.cs b/StabilityMatrix.Core/Models/Packages/Fooocus.cs index 93a717f1..a08a246e 100644 --- a/StabilityMatrix.Core/Models/Packages/Fooocus.cs +++ b/StabilityMatrix.Core/Models/Packages/Fooocus.cs @@ -377,7 +377,11 @@ private async Task SetupModelFoldersConfig(DirectoryPath installDirectory) installDirectory.JoinDir(OutputFolderName).Create(); await SharedFoldersConfigHelper - .UpdateJsonConfigFileForDefaultAsync(SharedFolderLayout, installDirectory) + .UpdateJsonConfigFileForSharedAsync( + SharedFolderLayout, + installDirectory, + SettingsManager.ModelsDirectory + ) .ConfigureAwait(false); } From 23e9456de9d3d8fec5e8dd0093e8e04cbb63b45d Mon Sep 17 00:00:00 2001 From: Ionite Date: Mon, 17 Jun 2024 21:39:49 -0400 Subject: [PATCH 19/25] Combined some methods --- .../Helper/SharedFoldersConfigHelper.cs | 103 +++++------------- 1 file changed, 26 insertions(+), 77 deletions(-) diff --git a/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs b/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs index f6c7ea22..3d9f86b5 100644 --- a/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs +++ b/StabilityMatrix.Core/Helper/SharedFoldersConfigHelper.cs @@ -7,6 +7,10 @@ namespace StabilityMatrix.Core.Helper; public static class SharedFoldersConfigHelper { + /// + /// Updates a JSON object with shared folder layout rules, using the SourceTypes, + /// converted to absolute paths using the sharedModelsDirectory. + /// public static async Task UpdateJsonConfigFileForSharedAsync( SharedFolderLayout layout, string packageRootDirectory, @@ -21,63 +25,22 @@ public static async Task UpdateJsonConfigFileForSharedAsync( await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); - await UpdateJsonConfigFileForSharedAsync(layout, stream, sharedModelsDirectory, options) - .ConfigureAwait(false); - } - - private static async Task UpdateJsonConfigFileForSharedAsync( - SharedFolderLayout layout, - Stream configStream, - string sharedModelsDirectory, - SharedFoldersConfigOptions? options = null - ) - { - options ??= SharedFoldersConfigOptions.Default; - - JsonObject jsonNode; - - if (configStream.Length == 0) - { - jsonNode = new JsonObject(); - } - else - { - jsonNode = - await JsonSerializer - .DeserializeAsync(configStream, options.JsonSerializerOptions) - .ConfigureAwait(false) ?? new JsonObject(); - } - - UpdateJsonConfigForShared(layout, jsonNode, sharedModelsDirectory, options); - - configStream.Seek(0, SeekOrigin.Begin); - configStream.SetLength(0); - - await JsonSerializer - .SerializeAsync(configStream, jsonNode, options.JsonSerializerOptions) + await UpdateJsonConfigFileAsync( + layout, + stream, + rule => + rule.SourceTypes.Select( + type => Path.Combine(sharedModelsDirectory, type.GetStringValue()) + ), + options + ) .ConfigureAwait(false); } /// - /// Updates a JSON object with shared folder layout rules, using the SourceTypes, - /// converted to absolute paths using the sharedModelsDirectory. + /// Updates a JSON object with shared folder layout rules, using the TargetRelativePaths, + /// converted to absolute paths using the packageRootDirectory. /// - private static void UpdateJsonConfigForShared( - SharedFolderLayout layout, - JsonObject jsonObject, - string sharedModelsDirectory, - SharedFoldersConfigOptions? options = null - ) - { - UpdateJsonConfig( - layout, - jsonObject, - rule => - rule.SourceTypes.Select(type => Path.Combine(sharedModelsDirectory, type.GetStringValue())), - options - ); - } - public static async Task UpdateJsonConfigFileForDefaultAsync( SharedFolderLayout layout, string packageRootDirectory, @@ -91,14 +54,21 @@ public static async Task UpdateJsonConfigFileForDefaultAsync( await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); - await UpdateJsonConfigFileForDefaultAsync(layout, stream, packageRootDirectory, options) + await UpdateJsonConfigFileAsync( + layout, + stream, + rule => + rule.TargetRelativePaths.Select(NormalizePathSlashes) + .Select(path => Path.Combine(packageRootDirectory, path)), + options + ) .ConfigureAwait(false); } - private static async Task UpdateJsonConfigFileForDefaultAsync( + private static async Task UpdateJsonConfigFileAsync( SharedFolderLayout layout, Stream configStream, - string packageRootDirectory, + Func> pathsSelector, SharedFoldersConfigOptions? options = null ) { @@ -118,7 +88,7 @@ await JsonSerializer .ConfigureAwait(false) ?? new JsonObject(); } - UpdateJsonConfigForDefault(layout, jsonNode, packageRootDirectory, options); + UpdateJsonConfig(layout, jsonNode, pathsSelector, options); configStream.Seek(0, SeekOrigin.Begin); configStream.SetLength(0); @@ -128,27 +98,6 @@ await JsonSerializer .ConfigureAwait(false); } - /// - /// Updates a JSON object with shared folder layout rules, using the TargetRelativePaths, - /// converted to absolute paths using the packageRootDirectory. - /// - private static void UpdateJsonConfigForDefault( - SharedFolderLayout layout, - JsonObject jsonObject, - string packageRootDirectory, - SharedFoldersConfigOptions? options = null - ) - { - UpdateJsonConfig( - layout, - jsonObject, - rule => - rule.TargetRelativePaths.Select(NormalizePathSlashes) - .Select(path => Path.Combine(packageRootDirectory, path)), - options - ); - } - private static void UpdateJsonConfig( SharedFolderLayout layout, JsonObject jsonObject, From 5f8c1cdd874b1d6f699678f763d52ff84507fe57 Mon Sep 17 00:00:00 2001 From: JT Date: Mon, 17 Jun 2024 18:46:47 -0700 Subject: [PATCH 20/25] Add disclaimer to FooocusControlNet since it looks ded --- StabilityMatrix.Core/Models/Packages/FocusControlNet.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs b/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs index f8fa9a19..f4c6af22 100644 --- a/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs +++ b/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs @@ -17,11 +17,14 @@ IPrerequisiteHelper prerequisiteHelper public override string DisplayName { get; set; } = "Fooocus-ControlNet"; public override string Author => "fenneishi"; public override string Blurb => "Fooocus-ControlNet adds more control to the original Fooocus software."; + public override string Disclaimer => + "This package has not been updated in over 8 months and may be abandoned."; + public override string LicenseUrl => "https://github.com/fenneishi/Fooocus-ControlNet-SDXL/blob/main/LICENSE"; public override Uri PreviewImageUri => new("https://github.com/fenneishi/Fooocus-ControlNet-SDXL/raw/main/asset/canny/snip.png"); - public override PackageDifficulty InstallerSortOrder => PackageDifficulty.Expert; + public override PackageDifficulty InstallerSortOrder => PackageDifficulty.Impossible; public override bool OfferInOneClickInstaller => false; public override SharedFolderLayout SharedFolderLayout => From a97646dfc64ef469e29cb99089467c349e57f362 Mon Sep 17 00:00:00 2001 From: JT Date: Mon, 17 Jun 2024 18:49:45 -0700 Subject: [PATCH 21/25] update Disclaimer --- StabilityMatrix.Core/Models/Packages/FocusControlNet.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs b/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs index f4c6af22..e610ddd4 100644 --- a/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs +++ b/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs @@ -17,9 +17,7 @@ IPrerequisiteHelper prerequisiteHelper public override string DisplayName { get; set; } = "Fooocus-ControlNet"; public override string Author => "fenneishi"; public override string Blurb => "Fooocus-ControlNet adds more control to the original Fooocus software."; - public override string Disclaimer => - "This package has not been updated in over 8 months and may be abandoned."; - + public override string Disclaimer => "This package may no longer be actively maintained"; public override string LicenseUrl => "https://github.com/fenneishi/Fooocus-ControlNet-SDXL/blob/main/LICENSE"; public override Uri PreviewImageUri => From 703f0a09afcb4bf6123d9a27d3ab9001ccb90fe7 Mon Sep 17 00:00:00 2001 From: Ionite Date: Tue, 18 Jun 2024 18:19:41 -0400 Subject: [PATCH 22/25] Add PyBaseInstall and CreatePyVenvRunner with tcl tk parsing --- StabilityMatrix.Core/Python/PyBaseInstall.cs | 132 ++++++++++++++++++ StabilityMatrix.Core/Python/PyVenvRunner.cs | 6 + .../Python/QueryTclTkLibraryResult.cs | 8 ++ 3 files changed, 146 insertions(+) create mode 100644 StabilityMatrix.Core/Python/PyBaseInstall.cs create mode 100644 StabilityMatrix.Core/Python/QueryTclTkLibraryResult.cs diff --git a/StabilityMatrix.Core/Python/PyBaseInstall.cs b/StabilityMatrix.Core/Python/PyBaseInstall.cs new file mode 100644 index 00000000..df11e4db --- /dev/null +++ b/StabilityMatrix.Core/Python/PyBaseInstall.cs @@ -0,0 +1,132 @@ +using System.Collections.Immutable; +using System.Text.Json; +using NLog; +using StabilityMatrix.Core.Exceptions; +using StabilityMatrix.Core.Helper; +using StabilityMatrix.Core.Models; +using StabilityMatrix.Core.Models.FileInterfaces; +using StabilityMatrix.Core.Processes; + +namespace StabilityMatrix.Core.Python; + +public class PyBaseInstall(DirectoryPath rootPath) +{ + private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); + + /// + /// Root path of the Python installation. + /// + public DirectoryPath RootPath { get; } = rootPath; + + /// + /// Whether this is a portable Windows installation. + /// Path structure is different. + /// + public bool IsWindowsPortable { get; init; } + + private int MajorVersion { get; init; } + + private int MinorVersion { get; init; } + + public FilePath PythonExePath => + Compat.Switch( + (PlatformKind.Windows, RootPath.JoinFile("python.exe")), + (PlatformKind.Linux, RootPath.JoinFile("bin", "python3")), + (PlatformKind.MacOS, RootPath.JoinFile("bin", "python3")) + ); + + public string DefaultTclTkPath => + Compat.Switch( + (PlatformKind.Windows, RootPath.JoinFile("tcl", "tcl8.6")), + (PlatformKind.Linux, RootPath.JoinFile("lib", "tcl8.6")), + (PlatformKind.MacOS, RootPath.JoinFile("lib", "tcl8.6")) + ); + + /// + /// Creates a new virtual environment runner. + /// + /// Root path of the venv + public PyVenvRunner CreateVenvRunner(DirectoryPath venvPath) + { + return new PyVenvRunner(RootPath, venvPath); + } + + /// + /// Creates a new virtual environment runner. + /// + /// Root path of the venv + /// Whether to include the Tcl/Tk library paths via + public async Task CreateVenvRunnerAsync(DirectoryPath venvPath, bool withTclTkEnv = false) + { + var runner = CreateVenvRunner(venvPath); + + if (withTclTkEnv) + { + var queryResult = await TryQueryTclTkLibraryAsync().ConfigureAwait(false); + if (queryResult is { Result: { } result }) + { + var env = + runner.EnvironmentVariables?.ToImmutableDictionary() + ?? ImmutableDictionary.Empty; + + if (!string.IsNullOrEmpty(result.TclLibrary)) + { + env = env.SetItem("TCL_LIBRARY", result.TclLibrary); + } + if (!string.IsNullOrEmpty(result.TkLibrary)) + { + env = env.SetItem("TK_LIBRARY", result.TkLibrary); + } + + runner.EnvironmentVariables = env; + } + else + { + Logger.Error(queryResult.Exception, "Failed to query Tcl/Tk library paths"); + } + } + + return runner; + } + + public async Task> TryQueryTclTkLibraryAsync() + { + var processResult = await QueryTclTkLibraryPathAsync().ConfigureAwait(false); + + if (!processResult.IsSuccessExitCode || string.IsNullOrEmpty(processResult.StandardOutput)) + { + return TaskResult.FromException(new ProcessException(processResult)); + } + + try + { + var result = JsonSerializer.Deserialize( + processResult.StandardOutput, + QueryTclTkLibraryResultJsonContext.Default.QueryTclTkLibraryResult + ); + + return new TaskResult(result!); + } + catch (JsonException e) + { + return TaskResult.FromException(e); + } + } + + private async Task QueryTclTkLibraryPathAsync() + { + const string script = """ + import tkinter + import json + + root = tkinter.Tk() + + print(json.dumps({ + 'TclLibrary': root.tk.exprstring('$tcl_library'), + 'TkLibrary': root.tk.exprstring('$tk_library') + })) + """; + + return await ProcessRunner.GetProcessResultAsync(PythonExePath, ["-c", script]).ConfigureAwait(false); + } +} diff --git a/StabilityMatrix.Core/Python/PyVenvRunner.cs b/StabilityMatrix.Core/Python/PyVenvRunner.cs index 4b13da5f..5b0d15ed 100644 --- a/StabilityMatrix.Core/Python/PyVenvRunner.cs +++ b/StabilityMatrix.Core/Python/PyVenvRunner.cs @@ -91,6 +91,12 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable /// public List SuppressOutput { get; } = new() { "fatal: not a git repository" }; + internal PyVenvRunner(DirectoryPath baseInstallRootPath, DirectoryPath rootPath) + { + RootPath = rootPath; + } + + [Obsolete("Use `PyBaseInstall.CreateVenvRunner` instead.")] public PyVenvRunner(DirectoryPath rootPath) { RootPath = rootPath; diff --git a/StabilityMatrix.Core/Python/QueryTclTkLibraryResult.cs b/StabilityMatrix.Core/Python/QueryTclTkLibraryResult.cs new file mode 100644 index 00000000..417bde0a --- /dev/null +++ b/StabilityMatrix.Core/Python/QueryTclTkLibraryResult.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization; + +namespace StabilityMatrix.Core.Python; + +public record QueryTclTkLibraryResult(string? TclLibrary, string? TkLibrary); + +[JsonSerializable(typeof(QueryTclTkLibraryResult))] +internal partial class QueryTclTkLibraryResultJsonContext : JsonSerializerContext; From a2f7c75cf1fedee43cb7c38431a08abc74d92c05 Mon Sep 17 00:00:00 2001 From: Ionite Date: Tue, 18 Jun 2024 20:12:00 -0400 Subject: [PATCH 23/25] Updated PyVenvRunner init for tk tcl and custom base python support. Refactors for packages. --- .../Models/PackageModification/PipStep.cs | 11 ++- .../Models/Packages/A3WebUI.cs | 5 +- .../Models/Packages/BaseGitPackage.cs | 67 +++++---------- .../Models/Packages/ComfyUI.cs | 5 +- .../Models/Packages/Fooocus.cs | 3 +- .../Models/Packages/FooocusMre.cs | 3 +- .../Models/Packages/InvokeAI.cs | 42 ++++----- .../Models/Packages/KohyaSs.cs | 10 +-- .../Models/Packages/OneTrainer.cs | 7 +- .../Models/Packages/RuinedFooocus.cs | 4 +- .../Models/Packages/SDWebForge.cs | 10 +-- StabilityMatrix.Core/Models/Packages/Sdfx.cs | 24 ++---- .../Packages/StableDiffusionDirectMl.cs | 6 +- .../Models/Packages/StableDiffusionUx.cs | 6 +- .../Models/Packages/VladAutomatic.cs | 13 +-- .../Models/Packages/VoltaML.cs | 7 +- StabilityMatrix.Core/Python/PyBaseInstall.cs | 86 +++++++++++++++---- StabilityMatrix.Core/Python/PyVenvRunner.cs | 41 ++++++--- 18 files changed, 169 insertions(+), 181 deletions(-) diff --git a/StabilityMatrix.Core/Models/PackageModification/PipStep.cs b/StabilityMatrix.Core/Models/PackageModification/PipStep.cs index 6d5d0cfa..3c31f134 100644 --- a/StabilityMatrix.Core/Models/PackageModification/PipStep.cs +++ b/StabilityMatrix.Core/Models/PackageModification/PipStep.cs @@ -1,5 +1,4 @@ using StabilityMatrix.Core.Extensions; -using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Models.FileInterfaces; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; @@ -29,11 +28,11 @@ _ when Args.Contains("-U") || Args.Contains("--upgrade") => "Updating Pip Packag /// public async Task ExecuteAsync(IProgress? progress = null) { - await using var venvRunner = new PyVenvRunner(VenvDirectory) - { - WorkingDirectory = WorkingDirectory, - EnvironmentVariables = EnvironmentVariables - }; + await using var venvRunner = PyBaseInstall.Default.CreateVenvRunner( + VenvDirectory, + workingDirectory: WorkingDirectory, + environmentVariables: EnvironmentVariables + ); var args = new List { "-m", "pip" }; args.AddRange(Args.ToArray()); diff --git a/StabilityMatrix.Core/Models/Packages/A3WebUI.cs b/StabilityMatrix.Core/Models/Packages/A3WebUI.cs index fb1510b8..6e1a83a2 100644 --- a/StabilityMatrix.Core/Models/Packages/A3WebUI.cs +++ b/StabilityMatrix.Core/Models/Packages/A3WebUI.cs @@ -204,11 +204,8 @@ public override async Task InstallPackage( var venvPath = Path.Combine(installLocation, "venv"); var exists = Directory.Exists(venvPath); - await using var venvRunner = new PyVenvRunner(venvPath); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); await venvRunner.PipInstall("--upgrade pip wheel", onConsoleOutput).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs index e368790d..ff1117a9 100644 --- a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs +++ b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs @@ -2,7 +2,6 @@ using System.IO.Compression; using NLog; using Octokit; -using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; using StabilityMatrix.Core.Models.Database; @@ -144,36 +143,26 @@ public async Task SetupVenv( Action? onConsoleOutput = null ) { - var venvPath = Path.Combine(installedPackagePath, venvName); if (VenvRunner != null) { await VenvRunner.DisposeAsync().ConfigureAwait(false); } - // Set additional required environment variables - var env = new Dictionary(); - if (SettingsManager.Settings.EnvironmentVariables is not null) - { - env.Update(SettingsManager.Settings.EnvironmentVariables); - } + VenvRunner = await PyBaseInstall + .Default.CreateVenvRunnerAsync( + Path.Combine(installedPackagePath, venvName), + workingDirectory: installedPackagePath, + environmentVariables: SettingsManager.Settings.EnvironmentVariables, + withDefaultTclTkEnv: Compat.IsWindows, + withQueriedTclTkEnv: Compat.IsUnix + ) + .ConfigureAwait(false); - if (Compat.IsWindows) + if (forceRecreate || !VenvRunner.Exists()) { - var tkPath = Path.Combine(SettingsManager.LibraryDir, "Assets", "Python310", "tcl", "tcl8.6"); - env["TCL_LIBRARY"] = tkPath; - env["TK_LIBRARY"] = tkPath; + await VenvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); } - VenvRunner = new PyVenvRunner(venvPath) - { - WorkingDirectory = installedPackagePath, - EnvironmentVariables = env - }; - - if (!VenvRunner.Exists() || forceRecreate) - { - await VenvRunner.Setup(forceRecreate, onConsoleOutput).ConfigureAwait(false); - } return VenvRunner; } @@ -188,31 +177,19 @@ public async Task SetupVenvPure( Action? onConsoleOutput = null ) { - var venvPath = Path.Combine(installedPackagePath, venvName); - - // Set additional required environment variables - var env = new Dictionary(); - if (SettingsManager.Settings.EnvironmentVariables is not null) - { - env.Update(SettingsManager.Settings.EnvironmentVariables); - } - - if (Compat.IsWindows) - { - var tkPath = Path.Combine(SettingsManager.LibraryDir, "Assets", "Python310", "tcl", "tcl8.6"); - env["TCL_LIBRARY"] = tkPath; - env["TK_LIBRARY"] = tkPath; - } - - var venvRunner = new PyVenvRunner(venvPath) - { - WorkingDirectory = installedPackagePath, - EnvironmentVariables = env - }; + var venvRunner = await PyBaseInstall + .Default.CreateVenvRunnerAsync( + Path.Combine(installedPackagePath, venvName), + workingDirectory: installedPackagePath, + environmentVariables: SettingsManager.Settings.EnvironmentVariables, + withDefaultTclTkEnv: Compat.IsWindows, + withQueriedTclTkEnv: Compat.IsUnix + ) + .ConfigureAwait(false); - if (!venvRunner.Exists() || forceRecreate) + if (forceRecreate || !venvRunner.Exists()) { - await venvRunner.Setup(forceRecreate, onConsoleOutput).ConfigureAwait(false); + await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); } return venvRunner; diff --git a/StabilityMatrix.Core/Models/Packages/ComfyUI.cs b/StabilityMatrix.Core/Models/Packages/ComfyUI.cs index 04effa62..6bdb0aed 100644 --- a/StabilityMatrix.Core/Models/Packages/ComfyUI.cs +++ b/StabilityMatrix.Core/Models/Packages/ComfyUI.cs @@ -193,11 +193,8 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1, "Setting up venv", isIndeterminate: true)); // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); await venvRunner.PipInstall("--upgrade pip wheel", onConsoleOutput).ConfigureAwait(false); progress?.Report( diff --git a/StabilityMatrix.Core/Models/Packages/Fooocus.cs b/StabilityMatrix.Core/Models/Packages/Fooocus.cs index a08a246e..905e9403 100644 --- a/StabilityMatrix.Core/Models/Packages/Fooocus.cs +++ b/StabilityMatrix.Core/Models/Packages/Fooocus.cs @@ -269,8 +269,7 @@ public override async Task InstallPackage( Action? onConsoleOutput = null ) { - var venvRunner = await SetupVenv(installLocation, forceRecreate: true).ConfigureAwait(false); - venvRunner.EnvironmentVariables = SettingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/FooocusMre.cs b/StabilityMatrix.Core/Models/Packages/FooocusMre.cs index 6d36d51c..1a1d1da7 100644 --- a/StabilityMatrix.Core/Models/Packages/FooocusMre.cs +++ b/StabilityMatrix.Core/Models/Packages/FooocusMre.cs @@ -110,8 +110,7 @@ public override async Task InstallPackage( Action? onConsoleOutput = null ) { - var venvRunner = await SetupVenv(installLocation, forceRecreate: true).ConfigureAwait(false); - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing torch...", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/InvokeAI.cs b/StabilityMatrix.Core/Models/Packages/InvokeAI.cs index b640187e..33fd6aba 100644 --- a/StabilityMatrix.Core/Models/Packages/InvokeAI.cs +++ b/StabilityMatrix.Core/Models/Packages/InvokeAI.cs @@ -1,5 +1,4 @@ -using System.Diagnostics; -using System.Globalization; +using System.Collections.Immutable; using System.Text.RegularExpressions; using NLog; using StabilityMatrix.Core.Attributes; @@ -162,11 +161,9 @@ public override async Task InstallPackage( var venvPath = Path.Combine(installLocation, "venv"); var exists = Directory.Exists(venvPath); - await using var venvRunner = new PyVenvRunner(venvPath); - venvRunner.WorkingDirectory = installLocation; - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); + venvRunner.UpdateEnvironmentVariables(env => GetEnvVars(env, installLocation)); - venvRunner.EnvironmentVariables = GetEnvVars(installLocation); progress?.Report(new ProgressReport(-1f, "Installing Package", isIndeterminate: true)); await SetupAndBuildInvokeFrontend( @@ -323,7 +320,7 @@ private async Task RunInvokeCommand( await SetupVenv(installedPackagePath).ConfigureAwait(false); - VenvRunner.EnvironmentVariables = GetEnvVars(installedPackagePath); + VenvRunner.UpdateEnvironmentVariables(env => GetEnvVars(env, installedPackagePath)); // fix frontend build missing for people who updated to v3.6 before the fix var frontendExistsPath = Path.Combine(installedPackagePath, RelativeFrontendBuildPath); @@ -410,44 +407,43 @@ void HandleConsoleOutput(ProcessOutput s) } } - private Dictionary GetEnvVars(DirectoryPath installPath) + private ImmutableDictionary GetEnvVars( + ImmutableDictionary env, + DirectoryPath installPath + ) { // Set additional required environment variables - var env = new Dictionary(); - if (SettingsManager.Settings.EnvironmentVariables is not null) - { - env.Update(SettingsManager.Settings.EnvironmentVariables); - } // Need to make subdirectory because they store config in the // directory *above* the root directory var root = installPath.JoinDir(RelativeRootPath); root.Create(); - env["INVOKEAI_ROOT"] = root; + env = env.SetItem("INVOKEAI_ROOT", root); - if (env.ContainsKey("PATH")) + var path = env.GetValueOrDefault("PATH", string.Empty); + + if (string.IsNullOrEmpty(path)) { - env["PATH"] += - $"{Compat.PathDelimiter}{Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs")}"; + path += $"{Compat.PathDelimiter}{Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs")}"; } else { - env["PATH"] = Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs"); + path += Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs"); } - env["PATH"] += $"{Compat.PathDelimiter}{Path.Combine(installPath, "node_modules", ".bin")}"; + + path += $"{Compat.PathDelimiter}{Path.Combine(installPath, "node_modules", ".bin")}"; if (Compat.IsMacOS || Compat.IsLinux) { - env["PATH"] += + path += $"{Compat.PathDelimiter}{Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs", "bin")}"; } if (Compat.IsWindows) { - env["PATH"] += - $"{Compat.PathDelimiter}{Environment.GetFolderPath(Environment.SpecialFolder.System)}"; + path += $"{Compat.PathDelimiter}{Environment.GetFolderPath(Environment.SpecialFolder.System)}"; } - return env; + return env.SetItem("PATH", path); } } diff --git a/StabilityMatrix.Core/Models/Packages/KohyaSs.cs b/StabilityMatrix.Core/Models/Packages/KohyaSs.cs index 3c493f83..b31c3363 100644 --- a/StabilityMatrix.Core/Models/Packages/KohyaSs.cs +++ b/StabilityMatrix.Core/Models/Packages/KohyaSs.cs @@ -125,11 +125,7 @@ await PrerequisiteHelper progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); // Extra dep needed before running setup since v23.0.x await venvRunner.PipInstall(["rich", "packaging"]).ConfigureAwait(false); @@ -156,7 +152,7 @@ public override async Task RunPackage( Action? onConsoleOutput ) { - var venvRunner = await SetupVenvPure(installedPackagePath).ConfigureAwait(false); + await SetupVenv(installedPackagePath).ConfigureAwait(false); void HandleConsoleOutput(ProcessOutput s) { @@ -176,7 +172,7 @@ void HandleConsoleOutput(ProcessOutput s) var args = $"\"{Path.Combine(installedPackagePath, command)}\" {arguments}"; - venvRunner.RunDetached(args.TrimEnd(), HandleConsoleOutput, OnExit); + VenvRunner.RunDetached(args.TrimEnd(), HandleConsoleOutput, OnExit); } public override Dictionary>? SharedFolders { get; } diff --git a/StabilityMatrix.Core/Models/Packages/OneTrainer.cs b/StabilityMatrix.Core/Models/Packages/OneTrainer.cs index 4a6887e6..52c49b35 100644 --- a/StabilityMatrix.Core/Models/Packages/OneTrainer.cs +++ b/StabilityMatrix.Core/Models/Packages/OneTrainer.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Text.RegularExpressions; using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; @@ -55,11 +54,7 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/RuinedFooocus.cs b/StabilityMatrix.Core/Models/Packages/RuinedFooocus.cs index 194b431e..f6def17b 100644 --- a/StabilityMatrix.Core/Models/Packages/RuinedFooocus.cs +++ b/StabilityMatrix.Core/Models/Packages/RuinedFooocus.cs @@ -115,8 +115,8 @@ public override async Task InstallPackage( { if (torchVersion == TorchVersion.Cuda) { - var venvRunner = await SetupVenv(installLocation, forceRecreate: true).ConfigureAwait(false); - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation, forceRecreate: true) + .ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/SDWebForge.cs b/StabilityMatrix.Core/Models/Packages/SDWebForge.cs index f2ef9962..9e304955 100644 --- a/StabilityMatrix.Core/Models/Packages/SDWebForge.cs +++ b/StabilityMatrix.Core/Models/Packages/SDWebForge.cs @@ -1,5 +1,4 @@ -using System.Text.Json.Nodes; -using StabilityMatrix.Core.Attributes; +using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; using StabilityMatrix.Core.Helper.HardwareInfo; @@ -156,13 +155,8 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); - var venvPath = Path.Combine(installLocation, "venv"); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); - await using var venvRunner = new PyVenvRunner(venvPath); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); await venvRunner.PipInstall("--upgrade pip wheel", onConsoleOutput).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/Sdfx.cs b/StabilityMatrix.Core/Models/Packages/Sdfx.cs index caced10c..6e6c579b 100644 --- a/StabilityMatrix.Core/Models/Packages/Sdfx.cs +++ b/StabilityMatrix.Core/Models/Packages/Sdfx.cs @@ -1,7 +1,7 @@ -using System.Text.Json; +using System.Collections.Immutable; +using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; -using NLog; using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Helper; @@ -9,7 +9,6 @@ using StabilityMatrix.Core.Models.FileInterfaces; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; -using StabilityMatrix.Core.Python; using StabilityMatrix.Core.Services; namespace StabilityMatrix.Core.Models.Packages; @@ -85,11 +84,8 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1, "Setting up venv", isIndeterminate: true)); // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = GetEnvVars(venvRunner, installLocation); - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); + venvRunner.UpdateEnvironmentVariables(GetEnvVars); progress?.Report( new ProgressReport(-1f, "Installing Package Requirements...", isIndeterminate: true) @@ -120,7 +116,7 @@ public override async Task RunPackage( ) { var venvRunner = await SetupVenv(installedPackagePath).ConfigureAwait(false); - venvRunner.EnvironmentVariables = GetEnvVars(venvRunner, installedPackagePath); + venvRunner.UpdateEnvironmentVariables(GetEnvVars); void HandleConsoleOutput(ProcessOutput s) { @@ -149,12 +145,8 @@ void HandleConsoleOutput(ProcessOutput s) } } - private Dictionary GetEnvVars(PyVenvRunner venvRunner, DirectoryPath installPath) + private ImmutableDictionary GetEnvVars(ImmutableDictionary env) { - var env = new Dictionary(); - env.Update(venvRunner.EnvironmentVariables ?? SettingsManager.Settings.EnvironmentVariables); - env["VIRTUAL_ENV"] = venvRunner.RootPath; - var pathBuilder = new EnvPathBuilder(); if (env.TryGetValue("PATH", out var value)) @@ -170,9 +162,7 @@ private Dictionary GetEnvVars(PyVenvRunner venvRunner, Directory pathBuilder.AddPath(Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs")); - env["PATH"] = pathBuilder.ToString(); - - return env; + return env.SetItem("PATH", pathBuilder.ToString()); } public override Task SetupModelFolders( diff --git a/StabilityMatrix.Core/Models/Packages/StableDiffusionDirectMl.cs b/StabilityMatrix.Core/Models/Packages/StableDiffusionDirectMl.cs index f6ad05c6..84b03d3c 100644 --- a/StabilityMatrix.Core/Models/Packages/StableDiffusionDirectMl.cs +++ b/StabilityMatrix.Core/Models/Packages/StableDiffusionDirectMl.cs @@ -74,11 +74,7 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); switch (torchVersion) { diff --git a/StabilityMatrix.Core/Models/Packages/StableDiffusionUx.cs b/StabilityMatrix.Core/Models/Packages/StableDiffusionUx.cs index 2b3d750e..f158c1b3 100644 --- a/StabilityMatrix.Core/Models/Packages/StableDiffusionUx.cs +++ b/StabilityMatrix.Core/Models/Packages/StableDiffusionUx.cs @@ -187,12 +187,8 @@ public override async Task InstallPackage( ) { progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); - // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); switch (torchVersion) { diff --git a/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs b/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs index f24f2acd..c99f390f 100644 --- a/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs +++ b/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs @@ -5,6 +5,7 @@ using System.Text.RegularExpressions; using NLog; using StabilityMatrix.Core.Attributes; +using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; using StabilityMatrix.Core.Helper.HardwareInfo; @@ -12,7 +13,6 @@ using StabilityMatrix.Core.Models.Packages.Extensions; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; -using StabilityMatrix.Core.Python; using StabilityMatrix.Core.Services; namespace StabilityMatrix.Core.Models.Packages; @@ -194,11 +194,7 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1f, "Installing package...", isIndeterminate: true)); // Setup venv - var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = SettingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); switch (torchVersion) { @@ -332,9 +328,8 @@ public override async Task Update( ) .ConfigureAwait(false); - var venvRunner = new PyVenvRunner(Path.Combine(installedPackage.FullPath!, "venv")); - venvRunner.WorkingDirectory = installedPackage.FullPath!; - venvRunner.EnvironmentVariables = SettingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installedPackage.FullPath!.Unwrap()) + .ConfigureAwait(false); await venvRunner.CustomInstall("launch.py --upgrade --test", onConsoleOutput).ConfigureAwait(false); diff --git a/StabilityMatrix.Core/Models/Packages/VoltaML.cs b/StabilityMatrix.Core/Models/Packages/VoltaML.cs index dba66a52..6286ed70 100644 --- a/StabilityMatrix.Core/Models/Packages/VoltaML.cs +++ b/StabilityMatrix.Core/Models/Packages/VoltaML.cs @@ -4,7 +4,6 @@ using StabilityMatrix.Core.Helper.Cache; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; -using StabilityMatrix.Core.Python; using StabilityMatrix.Core.Services; namespace StabilityMatrix.Core.Models.Packages; @@ -156,11 +155,7 @@ public override async Task InstallPackage( { // Setup venv progress?.Report(new ProgressReport(-1, "Setting up venv", isIndeterminate: true)); - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); // Install requirements progress?.Report(new ProgressReport(-1, "Installing Package Requirements", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Python/PyBaseInstall.cs b/StabilityMatrix.Core/Python/PyBaseInstall.cs index df11e4db..e0a8bd67 100644 --- a/StabilityMatrix.Core/Python/PyBaseInstall.cs +++ b/StabilityMatrix.Core/Python/PyBaseInstall.cs @@ -1,5 +1,4 @@ -using System.Collections.Immutable; -using System.Text.Json; +using System.Text.Json; using NLog; using StabilityMatrix.Core.Exceptions; using StabilityMatrix.Core.Helper; @@ -13,6 +12,8 @@ public class PyBaseInstall(DirectoryPath rootPath) { private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); + public static PyBaseInstall Default { get; } = new(PyRunner.PythonDir); + /// /// Root path of the Python installation. /// @@ -46,39 +47,87 @@ public class PyBaseInstall(DirectoryPath rootPath) /// Creates a new virtual environment runner. /// /// Root path of the venv - public PyVenvRunner CreateVenvRunner(DirectoryPath venvPath) + /// Working directory of the venv + /// Extra environment variables to set + /// Extra environment variables to set at the end + /// Whether to include the Tcl/Tk library paths via + public PyVenvRunner CreateVenvRunner( + DirectoryPath venvPath, + DirectoryPath? workingDirectory = null, + IReadOnlyDictionary? environmentVariables = null, + IReadOnlyDictionary? overrideEnvironmentVariables = null, + bool withDefaultTclTkEnv = false + ) { - return new PyVenvRunner(RootPath, venvPath); + var runner = new PyVenvRunner(this, venvPath) { WorkingDirectory = workingDirectory }; + + if (environmentVariables is { Count: > 0 }) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.AddRange(environmentVariables); + } + + if (withDefaultTclTkEnv) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.SetItem( + "TCL_LIBRARY", + DefaultTclTkPath + ); + runner.EnvironmentVariables = runner.EnvironmentVariables.SetItem("TK_LIBRARY", DefaultTclTkPath); + } + + if (overrideEnvironmentVariables is { Count: > 0 }) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.AddRange(overrideEnvironmentVariables); + } + + return runner; } /// /// Creates a new virtual environment runner. /// /// Root path of the venv - /// Whether to include the Tcl/Tk library paths via - public async Task CreateVenvRunnerAsync(DirectoryPath venvPath, bool withTclTkEnv = false) + /// Working directory of the venv + /// Extra environment variables to set + /// Extra environment variables to set at the end + /// Whether to include the Tcl/Tk library paths via + /// Whether to include the Tcl/Tk library paths via + public async Task CreateVenvRunnerAsync( + DirectoryPath venvPath, + DirectoryPath? workingDirectory = null, + IReadOnlyDictionary? environmentVariables = null, + IReadOnlyDictionary? overrideEnvironmentVariables = null, + bool withDefaultTclTkEnv = false, + bool withQueriedTclTkEnv = false + ) { - var runner = CreateVenvRunner(venvPath); + var runner = CreateVenvRunner( + venvPath: venvPath, + workingDirectory: workingDirectory, + environmentVariables: environmentVariables, + overrideEnvironmentVariables: null, + withDefaultTclTkEnv: withDefaultTclTkEnv + ); - if (withTclTkEnv) + if (withQueriedTclTkEnv) { var queryResult = await TryQueryTclTkLibraryAsync().ConfigureAwait(false); if (queryResult is { Result: { } result }) { - var env = - runner.EnvironmentVariables?.ToImmutableDictionary() - ?? ImmutableDictionary.Empty; - if (!string.IsNullOrEmpty(result.TclLibrary)) { - env = env.SetItem("TCL_LIBRARY", result.TclLibrary); + runner.EnvironmentVariables = runner.EnvironmentVariables.SetItem( + "TCL_LIBRARY", + result.TclLibrary + ); } if (!string.IsNullOrEmpty(result.TkLibrary)) { - env = env.SetItem("TK_LIBRARY", result.TkLibrary); + runner.EnvironmentVariables = runner.EnvironmentVariables.SetItem( + "TK_LIBRARY", + result.TkLibrary + ); } - - runner.EnvironmentVariables = env; } else { @@ -86,6 +135,11 @@ public async Task CreateVenvRunnerAsync(DirectoryPath venvPath, bo } } + if (overrideEnvironmentVariables is { Count: > 0 }) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.AddRange(overrideEnvironmentVariables); + } + return runner; } diff --git a/StabilityMatrix.Core/Python/PyVenvRunner.cs b/StabilityMatrix.Core/Python/PyVenvRunner.cs index 5b0d15ed..fe8678f3 100644 --- a/StabilityMatrix.Core/Python/PyVenvRunner.cs +++ b/StabilityMatrix.Core/Python/PyVenvRunner.cs @@ -1,4 +1,5 @@ -using System.Diagnostics.CodeAnalysis; +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; using System.Text; using System.Text.Json; using NLog; @@ -29,6 +30,8 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable (PlatformKind.Unix, "lib/python3.10/site-packages") ); + public PyBaseInstall BaseInstall { get; } + /// /// The process running the python executable. /// @@ -47,7 +50,8 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable /// /// Optional environment variables for the python process. /// - public IReadOnlyDictionary? EnvironmentVariables { get; set; } + public ImmutableDictionary EnvironmentVariables { get; set; } = + ImmutableDictionary.Empty; /// /// Name of the python binary folder. @@ -91,15 +95,25 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable /// public List SuppressOutput { get; } = new() { "fatal: not a git repository" }; - internal PyVenvRunner(DirectoryPath baseInstallRootPath, DirectoryPath rootPath) + internal PyVenvRunner(PyBaseInstall baseInstall, DirectoryPath rootPath) { + BaseInstall = baseInstall; RootPath = rootPath; + EnvironmentVariables = EnvironmentVariables.SetItem("VIRTUAL_ENV", rootPath.FullPath); } [Obsolete("Use `PyBaseInstall.CreateVenvRunner` instead.")] public PyVenvRunner(DirectoryPath rootPath) { RootPath = rootPath; + EnvironmentVariables = EnvironmentVariables.SetItem("VIRTUAL_ENV", rootPath.FullPath); + } + + public void UpdateEnvironmentVariables( + Func, ImmutableDictionary> env + ) + { + EnvironmentVariables = env(EnvironmentVariables); } /// True if the venv has a Scripts\python.exe file @@ -508,11 +522,7 @@ public void RunDetached( outputDataReceived.Invoke(s); }); - var env = new Dictionary(); - if (EnvironmentVariables != null) - { - env.Update(EnvironmentVariables); - } + var env = EnvironmentVariables; // Disable pip caching - uses significant memory for large packages like torch // env["PIP_NO_CACHE_DIR"] = "true"; @@ -524,29 +534,32 @@ public void RunDetached( var venvBin = RootPath.JoinDir(RelativeBinPath); if (env.TryGetValue("PATH", out var pathValue)) { - env["PATH"] = Compat.GetEnvPathWithExtensions(portableGitBin, venvBin, pathValue); + env = env.SetItem( + "PATH", + Compat.GetEnvPathWithExtensions(portableGitBin, venvBin, pathValue) + ); } else { - env["PATH"] = Compat.GetEnvPathWithExtensions(portableGitBin, venvBin); + env = env.SetItem("PATH", Compat.GetEnvPathWithExtensions(portableGitBin, venvBin)); } - env["GIT"] = portableGitBin.JoinFile("git.exe"); + env = env.SetItem("GIT", portableGitBin.JoinFile("git.exe")); } else { if (env.TryGetValue("PATH", out var pathValue)) { - env["PATH"] = Compat.GetEnvPathWithExtensions(pathValue); + env = env.SetItem("PATH", Compat.GetEnvPathWithExtensions(pathValue)); } else { - env["PATH"] = Compat.GetEnvPathWithExtensions(); + env = env.SetItem("PATH", Compat.GetEnvPathWithExtensions()); } } if (unbuffered) { - env["PYTHONUNBUFFERED"] = "1"; + env = env.SetItem("PYTHONUNBUFFERED", "1"); // If arguments starts with -, it's a flag, insert `u` after it for unbuffered mode if (arguments.StartsWith('-')) From d2131d21e3cb998611241523248fe740c87b44f1 Mon Sep 17 00:00:00 2001 From: Ionite Date: Tue, 18 Jun 2024 20:22:46 -0400 Subject: [PATCH 24/25] Fix kohya install arguments on linux --- StabilityMatrix.Core/Models/Packages/KohyaSs.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/StabilityMatrix.Core/Models/Packages/KohyaSs.cs b/StabilityMatrix.Core/Models/Packages/KohyaSs.cs index b31c3363..8e6c738f 100644 --- a/StabilityMatrix.Core/Models/Packages/KohyaSs.cs +++ b/StabilityMatrix.Core/Models/Packages/KohyaSs.cs @@ -134,13 +134,20 @@ await PrerequisiteHelper { // Install await venvRunner - .CustomInstall("setup/setup_windows.py --headless", onConsoleOutput) + .CustomInstall(["setup/setup_windows.py", "--headless"], onConsoleOutput) .ConfigureAwait(false); } else if (Compat.IsLinux) { await venvRunner - .CustomInstall("setup/setup_linux.py --headless", onConsoleOutput) + .CustomInstall( + [ + "setup/setup_linux.py", + "--platform-requirements-file=requirements_linux.txt", + "--no_run_accelerate" + ], + onConsoleOutput + ) .ConfigureAwait(false); } } From 2f6172c0fba5d020c0cadc1682429aa5ce0e0b45 Mon Sep 17 00:00:00 2001 From: Ionite Date: Tue, 18 Jun 2024 23:37:31 -0400 Subject: [PATCH 25/25] Update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0852c04d..741a4ac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,17 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2 - Added Rename option back to the Checkpoints page ### Changed - Unobserved Task Exceptions across the app will now show a toast notification to aid in debugging +- Updated SD.Next Package details and thumbnail - [#697](https://github.com/LykosAI/StabilityMatrix/pull/697) ### Fixed - Fixed [#689](https://github.com/LykosAI/StabilityMatrix/issues/689) - New ComfyUI installs encountering launch error due to torch 2.0.0 update, added pinned `numpy==1.26.4` to install and update. - Fixed Inference image mask editor's 'Load Mask' not able to load image files - Fixed Fooocus ControlNet default config shared folder mode not taking effect +- Fixed tkinter python libraries not working on macOS with 'Can't find a usable init.tcl' error +### Supporters +#### Visionaries +- Shoutout to our Visionary-tier supporters on Patreon, **Scopp Mcdee** and **Waterclouds**! Your generous support is appreciated and helps us continue to make Stability Matrix better for everyone! +#### Pioneers +- A big thank you to our Pioneer-tier supporters on Patreon, **tankfox** and **tanangular**! Your support helps us continue to improve Stability Matrix! ## v2.11.0 ### Added