diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 000000000..9e7038d52 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,39 @@ +name: 'Close stale issues' + +permissions: + issues: write + pull-requests: write + +on: + workflow_dispatch: + schedule: + - cron: '30 1 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: 'This issue is pending because it has been awaiting a response for 14 days with no activity. Remove the pending label or comment, else this will be closed in 5 days.' + close-issue-message: 'This issue was closed because it has been pending for 5 days with no activity.' + only-labels: 'awaiting-feedback' + stale-issue-label: 'pending' + exempt-issue-labels: 'planned,milestone,work-in-progress' + days-before-issue-stale: 14 + days-before-issue-close: 5 + days-before-pr-close: -1 + days-before-pr-stale: -1 + operations-per-run: 45 + + - uses: actions/stale@v9 + with: + stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove the stale label or comment, else this will be closed in 5 days.' + close-issue-message: 'This issue was closed because it has been stale for 5 days with no activity.' + stale-issue-label: 'stale' + exempt-issue-labels: 'planned,milestone,work-in-progress' + days-before-issue-stale: 30 + days-before-issue-close: 5 + days-before-pr-close: -1 + days-before-pr-stale: -1 + operations-per-run: 45 diff --git a/CHANGELOG.md b/CHANGELOG.md index e992dd7b9..741a4ac41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to Stability Matrix will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html). +## v2.11.1 +### Added +- Added Rename option back to the Checkpoints page +### Changed +- Unobserved Task Exceptions across the app will now show a toast notification to aid in debugging +- Updated SD.Next Package details and thumbnail - [#697](https://github.com/LykosAI/StabilityMatrix/pull/697) +### Fixed +- Fixed [#689](https://github.com/LykosAI/StabilityMatrix/issues/689) - New ComfyUI installs encountering launch error due to torch 2.0.0 update, added pinned `numpy==1.26.4` to install and update. +- Fixed Inference image mask editor's 'Load Mask' not able to load image files +- Fixed Fooocus ControlNet default config shared folder mode not taking effect +- Fixed tkinter python libraries not working on macOS with 'Can't find a usable init.tcl' error +### Supporters +#### Visionaries +- Shoutout to our Visionary-tier supporters on Patreon, **Scopp Mcdee** and **Waterclouds**! Your generous support is appreciated and helps us continue to make Stability Matrix better for everyone! +#### Pioneers +- A big thank you to our Pioneer-tier supporters on Patreon, **tankfox** and **tanangular**! Your support helps us continue to improve Stability Matrix! + ## v2.11.0 ### Added #### Packages diff --git a/StabilityMatrix.Avalonia/App.axaml.cs b/StabilityMatrix.Avalonia/App.axaml.cs index 88c380025..54e657542 100644 --- a/StabilityMatrix.Avalonia/App.axaml.cs +++ b/StabilityMatrix.Avalonia/App.axaml.cs @@ -294,6 +294,7 @@ private void ShowMainWindow() DesktopLifetime.ShutdownRequested += OnShutdownRequested; AppDomain.CurrentDomain.ProcessExit += OnExit; + TaskScheduler.UnobservedTaskException += TaskScheduler_UnobservedTaskException; // Since we're manually shutting down NLog in OnExit LogManager.AutoShutdown = false; @@ -883,6 +884,36 @@ private void OnExit(object? sender, EventArgs _) } } + private static void TaskScheduler_UnobservedTaskException( + object? sender, + UnobservedTaskExceptionEventArgs e + ) + { + if (e.Exception is not Exception unobservedEx) + return; + + try + { + var notificationService = Services.GetRequiredService(); + + Dispatcher.UIThread.Invoke(() => + { + var originException = unobservedEx.InnerException ?? unobservedEx; + notificationService.ShowPersistent( + $"Unobserved Task Exception - {originException.GetType().Name}", + originException.Message + ); + }); + + // Consider the exception observed if we were able to show a notification + e.SetObserved(); + } + catch (Exception ex) + { + Logger.Error(ex, "Failed to show Unobserved Task Exception notification"); + } + } + private static LoggingConfiguration ConfigureLogging() { var setupBuilder = LogManager.Setup(); diff --git a/StabilityMatrix.Avalonia/DesignData/DesignData.cs b/StabilityMatrix.Avalonia/DesignData/DesignData.cs index 89b561bb3..d2cbce9c1 100644 --- a/StabilityMatrix.Avalonia/DesignData/DesignData.cs +++ b/StabilityMatrix.Avalonia/DesignData/DesignData.cs @@ -327,6 +327,7 @@ public static void Initialize() new MockModelIndexService(), notificationService, dialogFactory, + null, new LocalModelFile { SharedFolderType = SharedFolderType.StableDiffusion, @@ -356,6 +357,7 @@ public static void Initialize() new MockModelIndexService(), notificationService, dialogFactory, + null, new LocalModelFile { RelativePath = "~/Models/Lora/model.safetensors", diff --git a/StabilityMatrix.Avalonia/Program.cs b/StabilityMatrix.Avalonia/Program.cs index 558914fb6..2f1488876 100644 --- a/StabilityMatrix.Avalonia/Program.cs +++ b/StabilityMatrix.Avalonia/Program.cs @@ -349,7 +349,7 @@ UnobservedTaskExceptionEventArgs e { if (e.Exception is Exception ex) { - Logger.Error(ex, "Unobserved task exception"); + Logger.Error(ex, "Unobserved Task Exception"); } } diff --git a/StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj b/StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj index 639930117..255e9a375 100644 --- a/StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj +++ b/StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj @@ -17,7 +17,7 @@ app.manifest true ./Assets/Icon.ico - 2.11.0-dev.999 + 2.12.0-dev.999 $(Version) true true diff --git a/StabilityMatrix.Avalonia/ViewModels/CheckpointManager/CheckpointFileViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/CheckpointManager/CheckpointFileViewModel.cs index 97095bc25..0264b9ca4 100644 --- a/StabilityMatrix.Avalonia/ViewModels/CheckpointManager/CheckpointFileViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/CheckpointManager/CheckpointFileViewModel.cs @@ -1,11 +1,14 @@ using System; using System.Collections.Immutable; using System.ComponentModel; +using System.IO; using System.Threading.Tasks; using Avalonia.Controls.Notifications; +using Avalonia.Data; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; using FluentAvalonia.UI.Controls; +using Microsoft.Extensions.Logging; using StabilityMatrix.Avalonia.Services; using StabilityMatrix.Avalonia.ViewModels.Base; using StabilityMatrix.Avalonia.ViewModels.Dialogs; @@ -36,6 +39,7 @@ public partial class CheckpointFileViewModel : SelectableViewModelBase private readonly IModelIndexService modelIndexService; private readonly INotificationService notificationService; private readonly ServiceManager vmFactory; + private readonly ILogger logger; public bool CanShowTriggerWords => CheckpointFile.ConnectedModelInfo?.TrainedWords?.Length > 0; public string BaseModelName => CheckpointFile.ConnectedModelInfo?.BaseModel ?? string.Empty; @@ -47,6 +51,7 @@ public CheckpointFileViewModel( IModelIndexService modelIndexService, INotificationService notificationService, ServiceManager vmFactory, + ILogger logger, LocalModelFile checkpointFile ) { @@ -54,6 +59,7 @@ LocalModelFile checkpointFile this.modelIndexService = modelIndexService; this.notificationService = notificationService; this.vmFactory = vmFactory; + this.logger = logger; CheckpointFile = checkpointFile; ThumbnailUri = settingsManager.IsLibraryDirSet ? CheckpointFile.GetPreviewImageFullPath(settingsManager.ModelsDirectory) @@ -186,4 +192,71 @@ private async Task DeleteAsync(bool showConfirmation = true) await modelIndexService.RemoveModelAsync(CheckpointFile); } + + [RelayCommand] + private async Task RenameAsync() + { + // Parent folder path + var parentPath = + Path.GetDirectoryName((string?)CheckpointFile.GetFullPath(settingsManager.ModelsDirectory)) ?? ""; + + var textFields = new TextBoxField[] + { + new() + { + Label = "File name", + Validator = text => + { + if (string.IsNullOrWhiteSpace(text)) + throw new DataValidationException("File name is required"); + + if (File.Exists(Path.Combine(parentPath, text))) + throw new DataValidationException("File name already exists"); + }, + Text = CheckpointFile.FileName + } + }; + + var dialog = DialogHelper.CreateTextEntryDialog("Rename Model", "", textFields); + + if (await dialog.ShowAsync() == ContentDialogResult.Primary) + { + var name = textFields[0].Text; + var nameNoExt = Path.GetFileNameWithoutExtension(name); + var originalNameNoExt = Path.GetFileNameWithoutExtension(CheckpointFile.FileName); + // Rename file in OS + try + { + var newFilePath = Path.Combine(parentPath, name); + File.Move(CheckpointFile.GetFullPath(settingsManager.ModelsDirectory), newFilePath); + + // If preview image exists, rename it too + var previewPath = CheckpointFile.GetPreviewImageFullPath(settingsManager.ModelsDirectory); + if (previewPath != null && File.Exists(previewPath)) + { + var newPreviewImagePath = Path.Combine( + parentPath, + $"{nameNoExt}.preview{Path.GetExtension(previewPath)}" + ); + File.Move(previewPath, newPreviewImagePath); + } + + // If connected model info exists, rename it too (.cm-info.json) + if (CheckpointFile.HasConnectedModel) + { + var cmInfoPath = Path.Combine(parentPath, $"{originalNameNoExt}.cm-info.json"); + if (File.Exists(cmInfoPath)) + { + File.Move(cmInfoPath, Path.Combine(parentPath, $"{nameNoExt}.cm-info.json")); + } + } + + await modelIndexService.RefreshIndex(); + } + catch (Exception e) + { + logger.LogError(e, "Failed to rename checkpoint file"); + } + } + } } diff --git a/StabilityMatrix.Avalonia/ViewModels/CheckpointsPageViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/CheckpointsPageViewModel.cs index a1aebc6e6..8ddac7b54 100644 --- a/StabilityMatrix.Avalonia/ViewModels/CheckpointsPageViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/CheckpointsPageViewModel.cs @@ -274,6 +274,7 @@ or nameof(SortConnectedModelsFirst) modelIndexService, notificationService, dialogFactory, + logger, x ) ) diff --git a/StabilityMatrix.Avalonia/ViewModels/Controls/PaintCanvasViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Controls/PaintCanvasViewModel.cs index fed45cebb..b94afe7c0 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Controls/PaintCanvasViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Controls/PaintCanvasViewModel.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.Collections.Immutable; using System.ComponentModel; using System.Linq; @@ -77,6 +76,9 @@ public partial class PaintCanvasViewModel(ILogger logger) [JsonIgnore] private SKLayer BrushLayer => Layers["Brush"]; + [JsonIgnore] + private SKLayer ImagesLayer => Layers["Images"]; + [JsonIgnore] private SKLayer BackgroundLayer => Layers["Background"]; @@ -99,9 +101,6 @@ public SKBitmap? BackgroundImage } } - [JsonIgnore] - public List LayerImages { get; } = []; - /// /// Set by to allow the view model to /// refresh the canvas view after updating points or bitmap layers. @@ -117,8 +116,7 @@ public void SetSourceCanvas(SKCanvas canvas) public void LoadCanvasFromBitmap(SKBitmap bitmap) { - LayerImages.Clear(); - LayerImages.Add(bitmap); + ImagesLayer.Bitmaps = [bitmap]; RefreshCanvas?.Invoke(); } diff --git a/StabilityMatrix.Avalonia/Views/CheckpointsPage.axaml b/StabilityMatrix.Avalonia/Views/CheckpointsPage.axaml index f6eb440ed..86b7a0f22 100644 --- a/StabilityMatrix.Avalonia/Views/CheckpointsPage.axaml +++ b/StabilityMatrix.Avalonia/Views/CheckpointsPage.axaml @@ -367,8 +367,8 @@ IsVisible="{Binding CheckpointFile.HasConnectedModel}" /> - - + + /// Updates a JSON object with shared folder layout rules, using the SourceTypes, + /// converted to absolute paths using the sharedModelsDirectory. + /// + public static async Task UpdateJsonConfigFileForSharedAsync( + SharedFolderLayout layout, + string packageRootDirectory, + string sharedModelsDirectory, + SharedFoldersConfigOptions? options = null + ) + { + var configPath = Path.Combine( + packageRootDirectory, + layout.RelativeConfigPath ?? throw new InvalidOperationException("RelativeConfigPath is null") + ); + + await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); + + await UpdateJsonConfigFileAsync( + layout, + stream, + rule => + rule.SourceTypes.Select( + type => Path.Combine(sharedModelsDirectory, type.GetStringValue()) + ), + options + ) + .ConfigureAwait(false); + } + + /// + /// Updates a JSON object with shared folder layout rules, using the TargetRelativePaths, + /// converted to absolute paths using the packageRootDirectory. + /// + public static async Task UpdateJsonConfigFileForDefaultAsync( + SharedFolderLayout layout, + string packageRootDirectory, + SharedFoldersConfigOptions? options = null + ) + { + var configPath = Path.Combine( + packageRootDirectory, + layout.RelativeConfigPath ?? throw new InvalidOperationException("RelativeConfigPath is null") + ); + + await using var stream = File.Open(configPath, FileMode.OpenOrCreate, FileAccess.ReadWrite); + + await UpdateJsonConfigFileAsync( + layout, + stream, + rule => + rule.TargetRelativePaths.Select(NormalizePathSlashes) + .Select(path => Path.Combine(packageRootDirectory, path)), + options + ) + .ConfigureAwait(false); + } + + private static async Task UpdateJsonConfigFileAsync( + SharedFolderLayout layout, + Stream configStream, + Func> pathsSelector, + SharedFoldersConfigOptions? options = null + ) + { + options ??= SharedFoldersConfigOptions.Default; + + JsonObject jsonNode; + + if (configStream.Length == 0) + { + jsonNode = new JsonObject(); + } + else + { + jsonNode = + await JsonSerializer + .DeserializeAsync(configStream, options.JsonSerializerOptions) + .ConfigureAwait(false) ?? new JsonObject(); + } + + UpdateJsonConfig(layout, jsonNode, pathsSelector, options); + + configStream.Seek(0, SeekOrigin.Begin); + configStream.SetLength(0); + + await JsonSerializer + .SerializeAsync(configStream, jsonNode, options.JsonSerializerOptions) + .ConfigureAwait(false); + } + + private static void UpdateJsonConfig( + SharedFolderLayout layout, + JsonObject jsonObject, + Func> pathsSelector, + SharedFoldersConfigOptions? options = null + ) + { + options ??= SharedFoldersConfigOptions.Default; + + var rulesByConfigPath = layout.GetRulesByConfigPath(); + + foreach (var (configPath, rule) in rulesByConfigPath) + { + // Get paths to write with selector + var paths = pathsSelector(rule).ToArray(); + + // Multiple elements or alwaysWriteArray is true, write as array + if (paths.Length > 1 || options.AlwaysWriteArray) + { + jsonObject[configPath] = new JsonArray( + paths.Select(path => (JsonNode)JsonValue.Create(path)).ToArray() + ); + } + // 1 element and alwaysWriteArray is false, write as string + else if (paths.Length == 1) + { + jsonObject[configPath] = paths[0]; + } + else + { + jsonObject.Remove(configPath); + } + } + } + + private static string NormalizePathSlashes(string path) + { + if (Compat.IsWindows) + { + return path.Replace('/', '\\'); + } + + return path; + } +} diff --git a/StabilityMatrix.Core/Helper/SharedFoldersConfigOptions.cs b/StabilityMatrix.Core/Helper/SharedFoldersConfigOptions.cs new file mode 100644 index 000000000..5cdb50487 --- /dev/null +++ b/StabilityMatrix.Core/Helper/SharedFoldersConfigOptions.cs @@ -0,0 +1,14 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StabilityMatrix.Core.Helper; + +public class SharedFoldersConfigOptions +{ + public static SharedFoldersConfigOptions Default => new(); + + public bool AlwaysWriteArray { get; set; } = false; + + public JsonSerializerOptions JsonSerializerOptions { get; set; } = + new() { WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault }; +} diff --git a/StabilityMatrix.Core/Models/PackageModification/PipStep.cs b/StabilityMatrix.Core/Models/PackageModification/PipStep.cs index 6d5d0cfac..3c31f1342 100644 --- a/StabilityMatrix.Core/Models/PackageModification/PipStep.cs +++ b/StabilityMatrix.Core/Models/PackageModification/PipStep.cs @@ -1,5 +1,4 @@ using StabilityMatrix.Core.Extensions; -using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Models.FileInterfaces; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; @@ -29,11 +28,11 @@ _ when Args.Contains("-U") || Args.Contains("--upgrade") => "Updating Pip Packag /// public async Task ExecuteAsync(IProgress? progress = null) { - await using var venvRunner = new PyVenvRunner(VenvDirectory) - { - WorkingDirectory = WorkingDirectory, - EnvironmentVariables = EnvironmentVariables - }; + await using var venvRunner = PyBaseInstall.Default.CreateVenvRunner( + VenvDirectory, + workingDirectory: WorkingDirectory, + environmentVariables: EnvironmentVariables + ); var args = new List { "-m", "pip" }; args.AddRange(Args.ToArray()); diff --git a/StabilityMatrix.Core/Models/Packages/A3WebUI.cs b/StabilityMatrix.Core/Models/Packages/A3WebUI.cs index fb1510b87..6e1a83a23 100644 --- a/StabilityMatrix.Core/Models/Packages/A3WebUI.cs +++ b/StabilityMatrix.Core/Models/Packages/A3WebUI.cs @@ -204,11 +204,8 @@ public override async Task InstallPackage( var venvPath = Path.Combine(installLocation, "venv"); var exists = Directory.Exists(venvPath); - await using var venvRunner = new PyVenvRunner(venvPath); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); await venvRunner.PipInstall("--upgrade pip wheel", onConsoleOutput).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs index e368790d4..ff1117a9b 100644 --- a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs +++ b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs @@ -2,7 +2,6 @@ using System.IO.Compression; using NLog; using Octokit; -using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; using StabilityMatrix.Core.Models.Database; @@ -144,36 +143,26 @@ public async Task SetupVenv( Action? onConsoleOutput = null ) { - var venvPath = Path.Combine(installedPackagePath, venvName); if (VenvRunner != null) { await VenvRunner.DisposeAsync().ConfigureAwait(false); } - // Set additional required environment variables - var env = new Dictionary(); - if (SettingsManager.Settings.EnvironmentVariables is not null) - { - env.Update(SettingsManager.Settings.EnvironmentVariables); - } + VenvRunner = await PyBaseInstall + .Default.CreateVenvRunnerAsync( + Path.Combine(installedPackagePath, venvName), + workingDirectory: installedPackagePath, + environmentVariables: SettingsManager.Settings.EnvironmentVariables, + withDefaultTclTkEnv: Compat.IsWindows, + withQueriedTclTkEnv: Compat.IsUnix + ) + .ConfigureAwait(false); - if (Compat.IsWindows) + if (forceRecreate || !VenvRunner.Exists()) { - var tkPath = Path.Combine(SettingsManager.LibraryDir, "Assets", "Python310", "tcl", "tcl8.6"); - env["TCL_LIBRARY"] = tkPath; - env["TK_LIBRARY"] = tkPath; + await VenvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); } - VenvRunner = new PyVenvRunner(venvPath) - { - WorkingDirectory = installedPackagePath, - EnvironmentVariables = env - }; - - if (!VenvRunner.Exists() || forceRecreate) - { - await VenvRunner.Setup(forceRecreate, onConsoleOutput).ConfigureAwait(false); - } return VenvRunner; } @@ -188,31 +177,19 @@ public async Task SetupVenvPure( Action? onConsoleOutput = null ) { - var venvPath = Path.Combine(installedPackagePath, venvName); - - // Set additional required environment variables - var env = new Dictionary(); - if (SettingsManager.Settings.EnvironmentVariables is not null) - { - env.Update(SettingsManager.Settings.EnvironmentVariables); - } - - if (Compat.IsWindows) - { - var tkPath = Path.Combine(SettingsManager.LibraryDir, "Assets", "Python310", "tcl", "tcl8.6"); - env["TCL_LIBRARY"] = tkPath; - env["TK_LIBRARY"] = tkPath; - } - - var venvRunner = new PyVenvRunner(venvPath) - { - WorkingDirectory = installedPackagePath, - EnvironmentVariables = env - }; + var venvRunner = await PyBaseInstall + .Default.CreateVenvRunnerAsync( + Path.Combine(installedPackagePath, venvName), + workingDirectory: installedPackagePath, + environmentVariables: SettingsManager.Settings.EnvironmentVariables, + withDefaultTclTkEnv: Compat.IsWindows, + withQueriedTclTkEnv: Compat.IsUnix + ) + .ConfigureAwait(false); - if (!venvRunner.Exists() || forceRecreate) + if (forceRecreate || !venvRunner.Exists()) { - await venvRunner.Setup(forceRecreate, onConsoleOutput).ConfigureAwait(false); + await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); } return venvRunner; diff --git a/StabilityMatrix.Core/Models/Packages/ComfyUI.cs b/StabilityMatrix.Core/Models/Packages/ComfyUI.cs index 8fa0f3c4e..6bdb0aedc 100644 --- a/StabilityMatrix.Core/Models/Packages/ComfyUI.cs +++ b/StabilityMatrix.Core/Models/Packages/ComfyUI.cs @@ -193,11 +193,8 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1, "Setting up venv", isIndeterminate: true)); // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); await venvRunner.PipInstall("--upgrade pip wheel", onConsoleOutput).ConfigureAwait(false); progress?.Report( @@ -231,12 +228,7 @@ public override async Task InstallPackage( ) }; - switch (torchVersion) - { - case TorchVersion.Mps: - pipArgs = pipArgs.AddArg("mpmath==1.3.0"); - break; - } + pipArgs = pipArgs.AddArg("numpy==1.26.4").AddArg("mpmath==1.3.0"); var requirements = new FilePath(installLocation, "requirements.txt"); diff --git a/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs b/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs index 9291c1e85..e610ddd4f 100644 --- a/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs +++ b/StabilityMatrix.Core/Models/Packages/FocusControlNet.cs @@ -1,12 +1,6 @@ -using System.Diagnostics; -using System.Text.RegularExpressions; -using StabilityMatrix.Core.Attributes; +using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; -using StabilityMatrix.Core.Models.FileInterfaces; -using StabilityMatrix.Core.Models.Progress; -using StabilityMatrix.Core.Processes; -using StabilityMatrix.Core.Python; using StabilityMatrix.Core.Services; namespace StabilityMatrix.Core.Models.Packages; @@ -23,9 +17,17 @@ IPrerequisiteHelper prerequisiteHelper public override string DisplayName { get; set; } = "Fooocus-ControlNet"; public override string Author => "fenneishi"; public override string Blurb => "Fooocus-ControlNet adds more control to the original Fooocus software."; - public override string LicenseUrl => "https://github.com/fenneishi/Fooocus-ControlNet-SDXL/blob/main/LICENSE"; + public override string Disclaimer => "This package may no longer be actively maintained"; + public override string LicenseUrl => + "https://github.com/fenneishi/Fooocus-ControlNet-SDXL/blob/main/LICENSE"; public override Uri PreviewImageUri => new("https://github.com/fenneishi/Fooocus-ControlNet-SDXL/raw/main/asset/canny/snip.png"); - public override PackageDifficulty InstallerSortOrder => PackageDifficulty.Expert; + public override PackageDifficulty InstallerSortOrder => PackageDifficulty.Impossible; public override bool OfferInOneClickInstaller => false; + + public override SharedFolderLayout SharedFolderLayout => + base.SharedFolderLayout with + { + RelativeConfigPath = "user_path_config.txt" + }; } diff --git a/StabilityMatrix.Core/Models/Packages/Fooocus.cs b/StabilityMatrix.Core/Models/Packages/Fooocus.cs index 3899fa0ef..905e94032 100644 --- a/StabilityMatrix.Core/Models/Packages/Fooocus.cs +++ b/StabilityMatrix.Core/Models/Packages/Fooocus.cs @@ -1,7 +1,4 @@ using System.Diagnostics; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.Json.Serialization; using System.Text.RegularExpressions; using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Helper; @@ -21,7 +18,9 @@ public class Fooocus( ISettingsManager settingsManager, IDownloadService downloadService, IPrerequisiteHelper prerequisiteHelper -) : BaseGitPackage(githubApi, settingsManager, downloadService, prerequisiteHelper) +) + : BaseGitPackage(githubApi, settingsManager, downloadService, prerequisiteHelper), + ISharedFolderLayoutPackage { public override string Name => "Fooocus"; public override string DisplayName { get; set; } = "Fooocus"; @@ -154,22 +153,100 @@ IPrerequisiteHelper prerequisiteHelper new[] { SharedFolderMethod.Symlink, SharedFolderMethod.Configuration, SharedFolderMethod.None }; public override Dictionary> SharedFolders => + ((ISharedFolderLayoutPackage)this).LegacySharedFolders; + + public virtual SharedFolderLayout SharedFolderLayout => new() { - [SharedFolderType.StableDiffusion] = new[] { "models/checkpoints" }, - [SharedFolderType.Diffusers] = new[] { "models/diffusers" }, - [SharedFolderType.Lora] = new[] { "models/loras" }, - [SharedFolderType.CLIP] = new[] { "models/clip" }, - [SharedFolderType.TextualInversion] = new[] { "models/embeddings" }, - [SharedFolderType.VAE] = new[] { "models/vae" }, - [SharedFolderType.ApproxVAE] = new[] { "models/vae_approx" }, - [SharedFolderType.ControlNet] = new[] { "models/controlnet" }, - [SharedFolderType.GLIGEN] = new[] { "models/gligen" }, - [SharedFolderType.ESRGAN] = new[] { "models/upscale_models" }, - [SharedFolderType.Hypernetwork] = new[] { "models/hypernetworks" } + RelativeConfigPath = "config.txt", + Rules = + [ + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.StableDiffusion], + TargetRelativePaths = ["models/checkpoints"], + ConfigDocumentPaths = ["path_checkpoints"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.Diffusers], + TargetRelativePaths = ["models/diffusers"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.CLIP], + TargetRelativePaths = ["models/clip"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.GLIGEN], + TargetRelativePaths = ["models/gligen"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.ESRGAN], + TargetRelativePaths = ["models/upscale_models"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.Hypernetwork], + TargetRelativePaths = ["models/hypernetworks"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.TextualInversion], + TargetRelativePaths = ["models/embeddings"], + ConfigDocumentPaths = ["path_embeddings"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.VAE], + TargetRelativePaths = ["models/vae"], + ConfigDocumentPaths = ["path_vae"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.ApproxVAE], + TargetRelativePaths = ["models/vae_approx"], + ConfigDocumentPaths = ["path_vae_approx"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.Lora, SharedFolderType.LyCORIS], + TargetRelativePaths = ["models/loras"], + ConfigDocumentPaths = ["path_loras"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.InvokeClipVision], + TargetRelativePaths = ["models/clip_vision"], + ConfigDocumentPaths = ["path_clip_vision"] + }, + new SharedFolderLayoutRule + { + SourceTypes = [SharedFolderType.ControlNet], + TargetRelativePaths = ["models/controlnet"], + ConfigDocumentPaths = ["path_controlnet"] + }, + new SharedFolderLayoutRule + { + TargetRelativePaths = ["models/inpaint"], + ConfigDocumentPaths = ["path_inpaint"] + }, + new SharedFolderLayoutRule + { + TargetRelativePaths = ["models/prompt_expansion/fooocus_expansion"], + ConfigDocumentPaths = ["path_fooocus_expansion"] + }, + new SharedFolderLayoutRule + { + TargetRelativePaths = [OutputFolderName], + ConfigDocumentPaths = ["path_outputs"] + } + ] }; - public override Dictionary>? SharedOutputFolders => + public override Dictionary> SharedOutputFolders => new() { [SharedOutputType.Text2Img] = new[] { "outputs" } }; public override IEnumerable AvailableTorchVersions => @@ -192,8 +269,7 @@ public override async Task InstallPackage( Action? onConsoleOutput = null ) { - var venvRunner = await SetupVenv(installLocation, forceRecreate: true).ConfigureAwait(false); - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); @@ -294,85 +370,28 @@ SharedFolderMethod sharedFolderMethod }; } - private JsonSerializerOptions jsonSerializerOptions = - new() { WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault }; - private async Task SetupModelFoldersConfig(DirectoryPath installDirectory) { - var fooocusConfigPath = installDirectory.JoinFile("config.txt"); - - var fooocusConfig = new JsonObject(); - - if (fooocusConfigPath.Exists) - { - fooocusConfig = - JsonSerializer.Deserialize( - await fooocusConfigPath.ReadAllTextAsync().ConfigureAwait(false) - ) ?? new JsonObject(); - } - - fooocusConfig["path_checkpoints"] = Path.Combine(settingsManager.ModelsDirectory, "StableDiffusion"); - fooocusConfig["path_loras"] = new JsonArray - { - Path.Combine(settingsManager.ModelsDirectory, "Lora"), - Path.Combine(settingsManager.ModelsDirectory, "LyCORIS") - }; - fooocusConfig["path_embeddings"] = Path.Combine(settingsManager.ModelsDirectory, "TextualInversion"); - fooocusConfig["path_vae_approx"] = Path.Combine(settingsManager.ModelsDirectory, "ApproxVAE"); - fooocusConfig["path_upscale_models"] = Path.Combine(settingsManager.ModelsDirectory, "ESRGAN"); - fooocusConfig["path_inpaint"] = Path.Combine(installDirectory, "models", "inpaint"); - fooocusConfig["path_controlnet"] = Path.Combine(settingsManager.ModelsDirectory, "ControlNet"); - fooocusConfig["path_clip_vision"] = Path.Combine(settingsManager.ModelsDirectory, "CLIP"); - fooocusConfig["path_fooocus_expansion"] = Path.Combine( - installDirectory, - "models", - "prompt_expansion", - "fooocus_expansion" - ); - - var outputsPath = Path.Combine(installDirectory, OutputFolderName); - // doesn't always exist on first install - Directory.CreateDirectory(outputsPath); - fooocusConfig["path_outputs"] = outputsPath; - - await fooocusConfigPath - .WriteAllTextAsync(JsonSerializer.Serialize(fooocusConfig, jsonSerializerOptions)) + installDirectory.JoinDir(OutputFolderName).Create(); + + await SharedFoldersConfigHelper + .UpdateJsonConfigFileForSharedAsync( + SharedFolderLayout, + installDirectory, + SettingsManager.ModelsDirectory + ) .ConfigureAwait(false); } - private async Task WriteDefaultConfig(DirectoryPath installDirectory) + private Task WriteDefaultConfig(DirectoryPath installDirectory) { - var fooocusConfigPath = installDirectory.JoinFile("config.txt"); - - var fooocusConfig = new JsonObject(); - - if (fooocusConfigPath.Exists) - { - fooocusConfig = - JsonSerializer.Deserialize( - await fooocusConfigPath.ReadAllTextAsync().ConfigureAwait(false) - ) ?? new JsonObject(); - } + // doesn't always exist on first install + installDirectory.JoinDir(OutputFolderName).Create(); - fooocusConfig["path_checkpoints"] = Path.Combine(installDirectory, "models", "checkpoints"); - fooocusConfig["path_loras"] = Path.Combine(installDirectory, "models", "loras"); - fooocusConfig["path_embeddings"] = Path.Combine(installDirectory, "models", "embeddings"); - fooocusConfig["path_vae_approx"] = Path.Combine(installDirectory, "models", "vae_approx"); - fooocusConfig["path_upscale_models"] = Path.Combine(installDirectory, "models", "upscale_models"); - fooocusConfig["path_inpaint"] = Path.Combine(installDirectory, "models", "inpaint"); - fooocusConfig["path_controlnet"] = Path.Combine(installDirectory, "models", "controlnet"); - fooocusConfig["path_clip_vision"] = Path.Combine(installDirectory, "models", "clip_vision"); - fooocusConfig["path_fooocus_expansion"] = Path.Combine( - installDirectory, - "models", - "prompt_expansion", - "fooocus_expansion" + return SharedFoldersConfigHelper.UpdateJsonConfigFileForDefaultAsync( + SharedFolderLayout, + installDirectory ); - fooocusConfig["path_outputs"] = Path.Combine(installDirectory, OutputFolderName); - - await fooocusConfigPath - .WriteAllTextAsync(JsonSerializer.Serialize(fooocusConfig, jsonSerializerOptions)) - .ConfigureAwait(false); } } diff --git a/StabilityMatrix.Core/Models/Packages/FooocusMre.cs b/StabilityMatrix.Core/Models/Packages/FooocusMre.cs index 6d36d51cc..1a1d1da72 100644 --- a/StabilityMatrix.Core/Models/Packages/FooocusMre.cs +++ b/StabilityMatrix.Core/Models/Packages/FooocusMre.cs @@ -110,8 +110,7 @@ public override async Task InstallPackage( Action? onConsoleOutput = null ) { - var venvRunner = await SetupVenv(installLocation, forceRecreate: true).ConfigureAwait(false); - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing torch...", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs b/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs new file mode 100644 index 000000000..58d1e9a09 --- /dev/null +++ b/StabilityMatrix.Core/Models/Packages/ISharedFolderLayoutPackage.cs @@ -0,0 +1,49 @@ +using System.Collections.Immutable; + +namespace StabilityMatrix.Core.Models.Packages; + +public interface ISharedFolderLayoutPackage +{ + SharedFolderLayout SharedFolderLayout { get; } + + Dictionary> LegacySharedFolders + { + get + { + // Keep track of unique paths since symbolic links can't do multiple targets + // So we'll ignore duplicates once they appear here + var addedPaths = new HashSet(); + var result = new Dictionary>(); + + foreach (var rule in SharedFolderLayout.Rules) + { + if (rule.TargetRelativePaths is not { Length: > 0 } value) + { + continue; + } + + foreach (var folderTypeKey in rule.SourceTypes) + { + var existingList = + (ImmutableList) + result.GetValueOrDefault(folderTypeKey, ImmutableList.Empty); + + foreach (var path in value) + { + // Skip if the path is already in the list + if (existingList.Contains(path)) + continue; + + // Skip if the path is already added globally + if (!addedPaths.Add(path)) + continue; + + result[folderTypeKey] = existingList.Add(path); + } + } + } + + return result; + } + } +} diff --git a/StabilityMatrix.Core/Models/Packages/InvokeAI.cs b/StabilityMatrix.Core/Models/Packages/InvokeAI.cs index b640187ec..33fd6aba7 100644 --- a/StabilityMatrix.Core/Models/Packages/InvokeAI.cs +++ b/StabilityMatrix.Core/Models/Packages/InvokeAI.cs @@ -1,5 +1,4 @@ -using System.Diagnostics; -using System.Globalization; +using System.Collections.Immutable; using System.Text.RegularExpressions; using NLog; using StabilityMatrix.Core.Attributes; @@ -162,11 +161,9 @@ public override async Task InstallPackage( var venvPath = Path.Combine(installLocation, "venv"); var exists = Directory.Exists(venvPath); - await using var venvRunner = new PyVenvRunner(venvPath); - venvRunner.WorkingDirectory = installLocation; - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); + venvRunner.UpdateEnvironmentVariables(env => GetEnvVars(env, installLocation)); - venvRunner.EnvironmentVariables = GetEnvVars(installLocation); progress?.Report(new ProgressReport(-1f, "Installing Package", isIndeterminate: true)); await SetupAndBuildInvokeFrontend( @@ -323,7 +320,7 @@ private async Task RunInvokeCommand( await SetupVenv(installedPackagePath).ConfigureAwait(false); - VenvRunner.EnvironmentVariables = GetEnvVars(installedPackagePath); + VenvRunner.UpdateEnvironmentVariables(env => GetEnvVars(env, installedPackagePath)); // fix frontend build missing for people who updated to v3.6 before the fix var frontendExistsPath = Path.Combine(installedPackagePath, RelativeFrontendBuildPath); @@ -410,44 +407,43 @@ void HandleConsoleOutput(ProcessOutput s) } } - private Dictionary GetEnvVars(DirectoryPath installPath) + private ImmutableDictionary GetEnvVars( + ImmutableDictionary env, + DirectoryPath installPath + ) { // Set additional required environment variables - var env = new Dictionary(); - if (SettingsManager.Settings.EnvironmentVariables is not null) - { - env.Update(SettingsManager.Settings.EnvironmentVariables); - } // Need to make subdirectory because they store config in the // directory *above* the root directory var root = installPath.JoinDir(RelativeRootPath); root.Create(); - env["INVOKEAI_ROOT"] = root; + env = env.SetItem("INVOKEAI_ROOT", root); - if (env.ContainsKey("PATH")) + var path = env.GetValueOrDefault("PATH", string.Empty); + + if (string.IsNullOrEmpty(path)) { - env["PATH"] += - $"{Compat.PathDelimiter}{Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs")}"; + path += $"{Compat.PathDelimiter}{Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs")}"; } else { - env["PATH"] = Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs"); + path += Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs"); } - env["PATH"] += $"{Compat.PathDelimiter}{Path.Combine(installPath, "node_modules", ".bin")}"; + + path += $"{Compat.PathDelimiter}{Path.Combine(installPath, "node_modules", ".bin")}"; if (Compat.IsMacOS || Compat.IsLinux) { - env["PATH"] += + path += $"{Compat.PathDelimiter}{Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs", "bin")}"; } if (Compat.IsWindows) { - env["PATH"] += - $"{Compat.PathDelimiter}{Environment.GetFolderPath(Environment.SpecialFolder.System)}"; + path += $"{Compat.PathDelimiter}{Environment.GetFolderPath(Environment.SpecialFolder.System)}"; } - return env; + return env.SetItem("PATH", path); } } diff --git a/StabilityMatrix.Core/Models/Packages/KohyaSs.cs b/StabilityMatrix.Core/Models/Packages/KohyaSs.cs index 3c493f833..8e6c738f0 100644 --- a/StabilityMatrix.Core/Models/Packages/KohyaSs.cs +++ b/StabilityMatrix.Core/Models/Packages/KohyaSs.cs @@ -125,11 +125,7 @@ await PrerequisiteHelper progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); // Extra dep needed before running setup since v23.0.x await venvRunner.PipInstall(["rich", "packaging"]).ConfigureAwait(false); @@ -138,13 +134,20 @@ await PrerequisiteHelper { // Install await venvRunner - .CustomInstall("setup/setup_windows.py --headless", onConsoleOutput) + .CustomInstall(["setup/setup_windows.py", "--headless"], onConsoleOutput) .ConfigureAwait(false); } else if (Compat.IsLinux) { await venvRunner - .CustomInstall("setup/setup_linux.py --headless", onConsoleOutput) + .CustomInstall( + [ + "setup/setup_linux.py", + "--platform-requirements-file=requirements_linux.txt", + "--no_run_accelerate" + ], + onConsoleOutput + ) .ConfigureAwait(false); } } @@ -156,7 +159,7 @@ public override async Task RunPackage( Action? onConsoleOutput ) { - var venvRunner = await SetupVenvPure(installedPackagePath).ConfigureAwait(false); + await SetupVenv(installedPackagePath).ConfigureAwait(false); void HandleConsoleOutput(ProcessOutput s) { @@ -176,7 +179,7 @@ void HandleConsoleOutput(ProcessOutput s) var args = $"\"{Path.Combine(installedPackagePath, command)}\" {arguments}"; - venvRunner.RunDetached(args.TrimEnd(), HandleConsoleOutput, OnExit); + VenvRunner.RunDetached(args.TrimEnd(), HandleConsoleOutput, OnExit); } public override Dictionary>? SharedFolders { get; } diff --git a/StabilityMatrix.Core/Models/Packages/OneTrainer.cs b/StabilityMatrix.Core/Models/Packages/OneTrainer.cs index 4a6887e65..52c49b355 100644 --- a/StabilityMatrix.Core/Models/Packages/OneTrainer.cs +++ b/StabilityMatrix.Core/Models/Packages/OneTrainer.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Text.RegularExpressions; using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; @@ -55,11 +54,7 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/RuinedFooocus.cs b/StabilityMatrix.Core/Models/Packages/RuinedFooocus.cs index 194b431e5..f6def17b4 100644 --- a/StabilityMatrix.Core/Models/Packages/RuinedFooocus.cs +++ b/StabilityMatrix.Core/Models/Packages/RuinedFooocus.cs @@ -115,8 +115,8 @@ public override async Task InstallPackage( { if (torchVersion == TorchVersion.Cuda) { - var venvRunner = await SetupVenv(installLocation, forceRecreate: true).ConfigureAwait(false); - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installLocation, forceRecreate: true) + .ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/SDWebForge.cs b/StabilityMatrix.Core/Models/Packages/SDWebForge.cs index f2ef99629..9e3049559 100644 --- a/StabilityMatrix.Core/Models/Packages/SDWebForge.cs +++ b/StabilityMatrix.Core/Models/Packages/SDWebForge.cs @@ -1,5 +1,4 @@ -using System.Text.Json.Nodes; -using StabilityMatrix.Core.Attributes; +using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; using StabilityMatrix.Core.Helper.HardwareInfo; @@ -156,13 +155,8 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); - var venvPath = Path.Combine(installLocation, "venv"); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); - await using var venvRunner = new PyVenvRunner(venvPath); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); await venvRunner.PipInstall("--upgrade pip wheel", onConsoleOutput).ConfigureAwait(false); progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Models/Packages/Sdfx.cs b/StabilityMatrix.Core/Models/Packages/Sdfx.cs index caced10c0..6e6c579b4 100644 --- a/StabilityMatrix.Core/Models/Packages/Sdfx.cs +++ b/StabilityMatrix.Core/Models/Packages/Sdfx.cs @@ -1,7 +1,7 @@ -using System.Text.Json; +using System.Collections.Immutable; +using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; -using NLog; using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Helper; @@ -9,7 +9,6 @@ using StabilityMatrix.Core.Models.FileInterfaces; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; -using StabilityMatrix.Core.Python; using StabilityMatrix.Core.Services; namespace StabilityMatrix.Core.Models.Packages; @@ -85,11 +84,8 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1, "Setting up venv", isIndeterminate: true)); // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = GetEnvVars(venvRunner, installLocation); - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); + venvRunner.UpdateEnvironmentVariables(GetEnvVars); progress?.Report( new ProgressReport(-1f, "Installing Package Requirements...", isIndeterminate: true) @@ -120,7 +116,7 @@ public override async Task RunPackage( ) { var venvRunner = await SetupVenv(installedPackagePath).ConfigureAwait(false); - venvRunner.EnvironmentVariables = GetEnvVars(venvRunner, installedPackagePath); + venvRunner.UpdateEnvironmentVariables(GetEnvVars); void HandleConsoleOutput(ProcessOutput s) { @@ -149,12 +145,8 @@ void HandleConsoleOutput(ProcessOutput s) } } - private Dictionary GetEnvVars(PyVenvRunner venvRunner, DirectoryPath installPath) + private ImmutableDictionary GetEnvVars(ImmutableDictionary env) { - var env = new Dictionary(); - env.Update(venvRunner.EnvironmentVariables ?? SettingsManager.Settings.EnvironmentVariables); - env["VIRTUAL_ENV"] = venvRunner.RootPath; - var pathBuilder = new EnvPathBuilder(); if (env.TryGetValue("PATH", out var value)) @@ -170,9 +162,7 @@ private Dictionary GetEnvVars(PyVenvRunner venvRunner, Directory pathBuilder.AddPath(Path.Combine(SettingsManager.LibraryDir, "Assets", "nodejs")); - env["PATH"] = pathBuilder.ToString(); - - return env; + return env.SetItem("PATH", pathBuilder.ToString()); } public override Task SetupModelFolders( diff --git a/StabilityMatrix.Core/Models/Packages/SharedFolderLayout.cs b/StabilityMatrix.Core/Models/Packages/SharedFolderLayout.cs new file mode 100644 index 000000000..2b121668a --- /dev/null +++ b/StabilityMatrix.Core/Models/Packages/SharedFolderLayout.cs @@ -0,0 +1,42 @@ +using System.Collections.Immutable; + +namespace StabilityMatrix.Core.Models.Packages; + +public record SharedFolderLayout +{ + /// + /// Optional config file path, relative from package installation directory + /// + public string? RelativeConfigPath { get; set; } + + public IImmutableList Rules { get; set; } = []; + + public Dictionary GetRulesByConfigPath() + { + // Dictionary of config path to rule + var configPathToRule = new Dictionary(); + + foreach (var rule in Rules) + { + // Ignore rules without config paths + if (rule.ConfigDocumentPaths is not { Length: > 0 } configPaths) + { + continue; + } + + foreach (var configPath in configPaths) + { + // Get or create rule + var existingRule = configPathToRule.GetValueOrDefault( + configPath, + new SharedFolderLayoutRule() + ); + + // Add unique + configPathToRule[configPath] = existingRule.Union(rule); + } + } + + return configPathToRule; + } +} diff --git a/StabilityMatrix.Core/Models/Packages/SharedFolderLayoutRule.cs b/StabilityMatrix.Core/Models/Packages/SharedFolderLayoutRule.cs new file mode 100644 index 000000000..8b795bc24 --- /dev/null +++ b/StabilityMatrix.Core/Models/Packages/SharedFolderLayoutRule.cs @@ -0,0 +1,40 @@ +namespace StabilityMatrix.Core.Models.Packages; + +public readonly record struct SharedFolderLayoutRule +{ + public SharedFolderType[] SourceTypes { get; init; } + + public string[] TargetRelativePaths { get; init; } + + public string[] ConfigDocumentPaths { get; init; } + + public SharedFolderLayoutRule() + { + SourceTypes = []; + TargetRelativePaths = []; + ConfigDocumentPaths = []; + } + + public SharedFolderLayoutRule(SharedFolderType[] types, string[] targets) + { + SourceTypes = types; + TargetRelativePaths = targets; + } + + public SharedFolderLayoutRule(SharedFolderType[] types, string[] targets, string[] configs) + { + SourceTypes = types; + TargetRelativePaths = targets; + ConfigDocumentPaths = configs; + } + + public SharedFolderLayoutRule Union(SharedFolderLayoutRule other) + { + return this with + { + SourceTypes = SourceTypes.Union(other.SourceTypes).ToArray(), + TargetRelativePaths = TargetRelativePaths.Union(other.TargetRelativePaths).ToArray(), + ConfigDocumentPaths = ConfigDocumentPaths.Union(other.ConfigDocumentPaths).ToArray() + }; + } +} diff --git a/StabilityMatrix.Core/Models/Packages/StableDiffusionDirectMl.cs b/StabilityMatrix.Core/Models/Packages/StableDiffusionDirectMl.cs index f6ad05c60..84b03d3cd 100644 --- a/StabilityMatrix.Core/Models/Packages/StableDiffusionDirectMl.cs +++ b/StabilityMatrix.Core/Models/Packages/StableDiffusionDirectMl.cs @@ -74,11 +74,7 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); switch (torchVersion) { diff --git a/StabilityMatrix.Core/Models/Packages/StableDiffusionUx.cs b/StabilityMatrix.Core/Models/Packages/StableDiffusionUx.cs index 2b3d750e5..f158c1b3f 100644 --- a/StabilityMatrix.Core/Models/Packages/StableDiffusionUx.cs +++ b/StabilityMatrix.Core/Models/Packages/StableDiffusionUx.cs @@ -187,12 +187,8 @@ public override async Task InstallPackage( ) { progress?.Report(new ProgressReport(-1f, "Setting up venv", isIndeterminate: true)); - // Setup venv - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); switch (torchVersion) { diff --git a/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs b/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs index 93063904b..a1570601a 100644 --- a/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs +++ b/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs @@ -5,6 +5,7 @@ using System.Text.RegularExpressions; using NLog; using StabilityMatrix.Core.Attributes; +using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Cache; using StabilityMatrix.Core.Helper.HardwareInfo; @@ -12,7 +13,6 @@ using StabilityMatrix.Core.Models.Packages.Extensions; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; -using StabilityMatrix.Core.Python; using StabilityMatrix.Core.Services; namespace StabilityMatrix.Core.Models.Packages; @@ -194,11 +194,7 @@ public override async Task InstallPackage( { progress?.Report(new ProgressReport(-1f, "Installing package...", isIndeterminate: true)); // Setup venv - var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = SettingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); switch (torchVersion) { @@ -332,9 +328,8 @@ public override async Task Update( ) .ConfigureAwait(false); - var venvRunner = new PyVenvRunner(Path.Combine(installedPackage.FullPath!, "venv")); - venvRunner.WorkingDirectory = installedPackage.FullPath!; - venvRunner.EnvironmentVariables = SettingsManager.Settings.EnvironmentVariables; + await using var venvRunner = await SetupVenvPure(installedPackage.FullPath!.Unwrap()) + .ConfigureAwait(false); await venvRunner.CustomInstall("launch.py --upgrade --test", onConsoleOutput).ConfigureAwait(false); diff --git a/StabilityMatrix.Core/Models/Packages/VoltaML.cs b/StabilityMatrix.Core/Models/Packages/VoltaML.cs index dba66a526..6286ed709 100644 --- a/StabilityMatrix.Core/Models/Packages/VoltaML.cs +++ b/StabilityMatrix.Core/Models/Packages/VoltaML.cs @@ -4,7 +4,6 @@ using StabilityMatrix.Core.Helper.Cache; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; -using StabilityMatrix.Core.Python; using StabilityMatrix.Core.Services; namespace StabilityMatrix.Core.Models.Packages; @@ -156,11 +155,7 @@ public override async Task InstallPackage( { // Setup venv progress?.Report(new ProgressReport(-1, "Setting up venv", isIndeterminate: true)); - await using var venvRunner = new PyVenvRunner(Path.Combine(installLocation, "venv")); - venvRunner.WorkingDirectory = installLocation; - venvRunner.EnvironmentVariables = settingsManager.Settings.EnvironmentVariables; - - await venvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await using var venvRunner = await SetupVenvPure(installLocation).ConfigureAwait(false); // Install requirements progress?.Report(new ProgressReport(-1, "Installing Package Requirements", isIndeterminate: true)); diff --git a/StabilityMatrix.Core/Python/PyBaseInstall.cs b/StabilityMatrix.Core/Python/PyBaseInstall.cs new file mode 100644 index 000000000..e0a8bd676 --- /dev/null +++ b/StabilityMatrix.Core/Python/PyBaseInstall.cs @@ -0,0 +1,186 @@ +using System.Text.Json; +using NLog; +using StabilityMatrix.Core.Exceptions; +using StabilityMatrix.Core.Helper; +using StabilityMatrix.Core.Models; +using StabilityMatrix.Core.Models.FileInterfaces; +using StabilityMatrix.Core.Processes; + +namespace StabilityMatrix.Core.Python; + +public class PyBaseInstall(DirectoryPath rootPath) +{ + private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); + + public static PyBaseInstall Default { get; } = new(PyRunner.PythonDir); + + /// + /// Root path of the Python installation. + /// + public DirectoryPath RootPath { get; } = rootPath; + + /// + /// Whether this is a portable Windows installation. + /// Path structure is different. + /// + public bool IsWindowsPortable { get; init; } + + private int MajorVersion { get; init; } + + private int MinorVersion { get; init; } + + public FilePath PythonExePath => + Compat.Switch( + (PlatformKind.Windows, RootPath.JoinFile("python.exe")), + (PlatformKind.Linux, RootPath.JoinFile("bin", "python3")), + (PlatformKind.MacOS, RootPath.JoinFile("bin", "python3")) + ); + + public string DefaultTclTkPath => + Compat.Switch( + (PlatformKind.Windows, RootPath.JoinFile("tcl", "tcl8.6")), + (PlatformKind.Linux, RootPath.JoinFile("lib", "tcl8.6")), + (PlatformKind.MacOS, RootPath.JoinFile("lib", "tcl8.6")) + ); + + /// + /// Creates a new virtual environment runner. + /// + /// Root path of the venv + /// Working directory of the venv + /// Extra environment variables to set + /// Extra environment variables to set at the end + /// Whether to include the Tcl/Tk library paths via + public PyVenvRunner CreateVenvRunner( + DirectoryPath venvPath, + DirectoryPath? workingDirectory = null, + IReadOnlyDictionary? environmentVariables = null, + IReadOnlyDictionary? overrideEnvironmentVariables = null, + bool withDefaultTclTkEnv = false + ) + { + var runner = new PyVenvRunner(this, venvPath) { WorkingDirectory = workingDirectory }; + + if (environmentVariables is { Count: > 0 }) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.AddRange(environmentVariables); + } + + if (withDefaultTclTkEnv) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.SetItem( + "TCL_LIBRARY", + DefaultTclTkPath + ); + runner.EnvironmentVariables = runner.EnvironmentVariables.SetItem("TK_LIBRARY", DefaultTclTkPath); + } + + if (overrideEnvironmentVariables is { Count: > 0 }) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.AddRange(overrideEnvironmentVariables); + } + + return runner; + } + + /// + /// Creates a new virtual environment runner. + /// + /// Root path of the venv + /// Working directory of the venv + /// Extra environment variables to set + /// Extra environment variables to set at the end + /// Whether to include the Tcl/Tk library paths via + /// Whether to include the Tcl/Tk library paths via + public async Task CreateVenvRunnerAsync( + DirectoryPath venvPath, + DirectoryPath? workingDirectory = null, + IReadOnlyDictionary? environmentVariables = null, + IReadOnlyDictionary? overrideEnvironmentVariables = null, + bool withDefaultTclTkEnv = false, + bool withQueriedTclTkEnv = false + ) + { + var runner = CreateVenvRunner( + venvPath: venvPath, + workingDirectory: workingDirectory, + environmentVariables: environmentVariables, + overrideEnvironmentVariables: null, + withDefaultTclTkEnv: withDefaultTclTkEnv + ); + + if (withQueriedTclTkEnv) + { + var queryResult = await TryQueryTclTkLibraryAsync().ConfigureAwait(false); + if (queryResult is { Result: { } result }) + { + if (!string.IsNullOrEmpty(result.TclLibrary)) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.SetItem( + "TCL_LIBRARY", + result.TclLibrary + ); + } + if (!string.IsNullOrEmpty(result.TkLibrary)) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.SetItem( + "TK_LIBRARY", + result.TkLibrary + ); + } + } + else + { + Logger.Error(queryResult.Exception, "Failed to query Tcl/Tk library paths"); + } + } + + if (overrideEnvironmentVariables is { Count: > 0 }) + { + runner.EnvironmentVariables = runner.EnvironmentVariables.AddRange(overrideEnvironmentVariables); + } + + return runner; + } + + public async Task> TryQueryTclTkLibraryAsync() + { + var processResult = await QueryTclTkLibraryPathAsync().ConfigureAwait(false); + + if (!processResult.IsSuccessExitCode || string.IsNullOrEmpty(processResult.StandardOutput)) + { + return TaskResult.FromException(new ProcessException(processResult)); + } + + try + { + var result = JsonSerializer.Deserialize( + processResult.StandardOutput, + QueryTclTkLibraryResultJsonContext.Default.QueryTclTkLibraryResult + ); + + return new TaskResult(result!); + } + catch (JsonException e) + { + return TaskResult.FromException(e); + } + } + + private async Task QueryTclTkLibraryPathAsync() + { + const string script = """ + import tkinter + import json + + root = tkinter.Tk() + + print(json.dumps({ + 'TclLibrary': root.tk.exprstring('$tcl_library'), + 'TkLibrary': root.tk.exprstring('$tk_library') + })) + """; + + return await ProcessRunner.GetProcessResultAsync(PythonExePath, ["-c", script]).ConfigureAwait(false); + } +} diff --git a/StabilityMatrix.Core/Python/PyVenvRunner.cs b/StabilityMatrix.Core/Python/PyVenvRunner.cs index 4b13da5fb..fe8678f3b 100644 --- a/StabilityMatrix.Core/Python/PyVenvRunner.cs +++ b/StabilityMatrix.Core/Python/PyVenvRunner.cs @@ -1,4 +1,5 @@ -using System.Diagnostics.CodeAnalysis; +using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; using System.Text; using System.Text.Json; using NLog; @@ -29,6 +30,8 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable (PlatformKind.Unix, "lib/python3.10/site-packages") ); + public PyBaseInstall BaseInstall { get; } + /// /// The process running the python executable. /// @@ -47,7 +50,8 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable /// /// Optional environment variables for the python process. /// - public IReadOnlyDictionary? EnvironmentVariables { get; set; } + public ImmutableDictionary EnvironmentVariables { get; set; } = + ImmutableDictionary.Empty; /// /// Name of the python binary folder. @@ -91,9 +95,25 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable /// public List SuppressOutput { get; } = new() { "fatal: not a git repository" }; + internal PyVenvRunner(PyBaseInstall baseInstall, DirectoryPath rootPath) + { + BaseInstall = baseInstall; + RootPath = rootPath; + EnvironmentVariables = EnvironmentVariables.SetItem("VIRTUAL_ENV", rootPath.FullPath); + } + + [Obsolete("Use `PyBaseInstall.CreateVenvRunner` instead.")] public PyVenvRunner(DirectoryPath rootPath) { RootPath = rootPath; + EnvironmentVariables = EnvironmentVariables.SetItem("VIRTUAL_ENV", rootPath.FullPath); + } + + public void UpdateEnvironmentVariables( + Func, ImmutableDictionary> env + ) + { + EnvironmentVariables = env(EnvironmentVariables); } /// True if the venv has a Scripts\python.exe file @@ -502,11 +522,7 @@ public void RunDetached( outputDataReceived.Invoke(s); }); - var env = new Dictionary(); - if (EnvironmentVariables != null) - { - env.Update(EnvironmentVariables); - } + var env = EnvironmentVariables; // Disable pip caching - uses significant memory for large packages like torch // env["PIP_NO_CACHE_DIR"] = "true"; @@ -518,29 +534,32 @@ public void RunDetached( var venvBin = RootPath.JoinDir(RelativeBinPath); if (env.TryGetValue("PATH", out var pathValue)) { - env["PATH"] = Compat.GetEnvPathWithExtensions(portableGitBin, venvBin, pathValue); + env = env.SetItem( + "PATH", + Compat.GetEnvPathWithExtensions(portableGitBin, venvBin, pathValue) + ); } else { - env["PATH"] = Compat.GetEnvPathWithExtensions(portableGitBin, venvBin); + env = env.SetItem("PATH", Compat.GetEnvPathWithExtensions(portableGitBin, venvBin)); } - env["GIT"] = portableGitBin.JoinFile("git.exe"); + env = env.SetItem("GIT", portableGitBin.JoinFile("git.exe")); } else { if (env.TryGetValue("PATH", out var pathValue)) { - env["PATH"] = Compat.GetEnvPathWithExtensions(pathValue); + env = env.SetItem("PATH", Compat.GetEnvPathWithExtensions(pathValue)); } else { - env["PATH"] = Compat.GetEnvPathWithExtensions(); + env = env.SetItem("PATH", Compat.GetEnvPathWithExtensions()); } } if (unbuffered) { - env["PYTHONUNBUFFERED"] = "1"; + env = env.SetItem("PYTHONUNBUFFERED", "1"); // If arguments starts with -, it's a flag, insert `u` after it for unbuffered mode if (arguments.StartsWith('-')) diff --git a/StabilityMatrix.Core/Python/QueryTclTkLibraryResult.cs b/StabilityMatrix.Core/Python/QueryTclTkLibraryResult.cs new file mode 100644 index 000000000..417bde0ae --- /dev/null +++ b/StabilityMatrix.Core/Python/QueryTclTkLibraryResult.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization; + +namespace StabilityMatrix.Core.Python; + +public record QueryTclTkLibraryResult(string? TclLibrary, string? TkLibrary); + +[JsonSerializable(typeof(QueryTclTkLibraryResult))] +internal partial class QueryTclTkLibraryResultJsonContext : JsonSerializerContext;