From f67b538042649b3e893361a388e1ca5d3c7b486d Mon Sep 17 00:00:00 2001 From: Tulili Date: Mon, 26 Feb 2024 00:41:52 -0300 Subject: [PATCH 1/7] fix(clean): a LOT faster cleanup code + no progress bar --- cmd/layer/clean/clean.go | 138 ++++++++++----------------------------- 1 file changed, 36 insertions(+), 102 deletions(-) diff --git a/cmd/layer/clean/clean.go b/cmd/layer/clean/clean.go index 734085e..1838022 100644 --- a/cmd/layer/clean/clean.go +++ b/cmd/layer/clean/clean.go @@ -6,13 +6,10 @@ import ( "os" "path" "path/filepath" - "slices" + "sync" - "github.com/jedib0t/go-pretty/v6/progress" "github.com/spf13/cobra" "github.com/ublue-os/bext/internal" - "github.com/ublue-os/bext/pkg/logging" - "github.com/ublue-os/bext/pkg/percentmanager" ) var CleanCmd = &cobra.Command{ @@ -32,28 +29,6 @@ func init() { fDryRun = CleanCmd.Flags().Bool("dry-run", false, "Do not actually clean anything, just print what would be deleted") } -func getWhatNotToClean(clean []string) ([]string, error) { - var do_not_clean []string - for _, cleanthing := range clean { - fstat, err := os.Lstat(cleanthing) - if err != nil { - return nil, err - } - if fstat.Mode().Type() == os.ModeSymlink && fstat.Name() == internal.CurrentBlobName { - eval_link, err := filepath.EvalSymlinks(cleanthing) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } else if errors.Is(err, os.ErrNotExist) { - continue - } - do_not_clean = append(do_not_clean, eval_link) - do_not_clean = append(do_not_clean, cleanthing) - break - } - } - return do_not_clean, nil -} - func cleanCmd(cmd *cobra.Command, args []string) error { cache_dir, err := filepath.Abs(internal.Config.CacheDir) if err != nil { @@ -64,14 +39,13 @@ func cleanCmd(cmd *cobra.Command, args []string) error { return err } - base_message := "Cleaning " - pw := percent.NewProgressWriter() + var wg sync.WaitGroup - var expectedSections int for _, entry := range target_cache { if !entry.IsDir() { continue } + slog.Info("Cleaning layer " + entry.Name()) entry_dir_path := path.Join(cache_dir, entry.Name()) entry_dir, err := os.ReadDir(entry_dir_path) @@ -79,101 +53,61 @@ func cleanCmd(cmd *cobra.Command, args []string) error { return err } - var clean []string - - for _, cache_blob := range entry_dir { - if cache_blob.IsDir() { - continue - } - - clean = append(clean, path.Join(entry_dir_path, cache_blob.Name())) - } - - do_not_clean, err := getWhatNotToClean(clean) - if err != nil { - return err - } - if len(entry_dir) < 1 { - expectedSections++ + wg.Add(1) + go func() { + wg.Done() + os.Remove(entry_dir_path) + }() continue } - for _, cache_blob := range entry_dir { - if cache_blob.IsDir() { - continue - } - expectedSections++ - } - expectedSections = expectedSections - len(do_not_clean) - } - - delete_tracker := percent.NewIncrementTracker(&progress.Tracker{Message: base_message, Total: int64(100), Units: progress.UnitsDefault}, expectedSections) - pw.AppendTracker(delete_tracker.Tracker) + var do_not_clean map[string]bool = make(map[string]bool) - if !*internal.Config.NoProgress { - go pw.Render() - slog.SetDefault(logging.NewMuteLogger()) - } - - for _, entry := range target_cache { - if !entry.IsDir() { - continue - } - logmessage := base_message + entry.Name() - delete_tracker.Tracker.Message = logmessage - slog.Info(logmessage) - - entry_dir_path := path.Join(cache_dir, entry.Name()) - entry_dir, err := os.ReadDir(entry_dir_path) - if err != nil { - delete_tracker.Tracker.MarkAsErrored() - return err - } - - if len(entry_dir) < 1 { - delete_tracker.IncrementSection() - os.Remove(entry_dir_path) - continue + for _, provided_path := range *fExclude { + managed_path, err := filepath.Abs(path.Clean(provided_path)) + if err != nil { + return err + } + do_not_clean[managed_path] = true } - var clean []string - for _, cache_blob := range entry_dir { if cache_blob.IsDir() { continue } - clean = append(clean, path.Join(entry_dir_path, cache_blob.Name())) - } - - do_not_clean, err := getWhatNotToClean(clean) - if err != nil { - delete_tracker.Tracker.MarkAsErrored() - return err - } + cleanpath := path.Join(entry_dir_path, cache_blob.Name()) - for _, provided_path := range *fExclude { - managed_path, err := filepath.Abs(path.Clean(provided_path)) + fstat, err := os.Lstat(cleanpath) if err != nil { return err } - do_not_clean = append(do_not_clean, managed_path) - } - for _, cleanthing := range clean { - slog.Debug("Clean -> " + cleanthing) - if slices.Contains(do_not_clean, cleanthing) || *fDryRun { + if fstat.Mode().Type() == os.ModeSymlink && fstat.Name() == internal.CurrentBlobName { + eval_link, err := filepath.EvalSymlinks(cleanpath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } else if errors.Is(err, os.ErrNotExist) { + continue + } + do_not_clean[eval_link] = true + do_not_clean[cleanpath] = true continue } - delete_tracker.IncrementSection() - if err := os.Remove(cleanthing); err != nil { - delete_tracker.Tracker.MarkAsErrored() - return err + if _, exists := do_not_clean[cleanpath]; exists || *fDryRun { + continue } + + slog.Debug("Cleaned path", slog.String("path", cleanpath)) + wg.Add(1) + go func() { + defer wg.Done() + os.Remove(cleanpath) + }() } } - delete_tracker.Tracker.MarkAsDone() + wg.Wait() return nil } From 55480c4ae8ae5060a26ba20293c353a8354851de Mon Sep 17 00:00:00 2001 From: Tulili Date: Mon, 26 Feb 2024 10:34:49 -0300 Subject: [PATCH 2/7] feat(activate): make it possible to activate multiple layers at once --- cmd/layer/activate/activate.go | 93 ++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/cmd/layer/activate/activate.go b/cmd/layer/activate/activate.go index a6e59c1..0d656ae 100644 --- a/cmd/layer/activate/activate.go +++ b/cmd/layer/activate/activate.go @@ -1,7 +1,7 @@ package activate import ( - "fmt" + "errors" "log/slog" "os" "path" @@ -13,50 +13,51 @@ import ( ) var ActivateCmd = &cobra.Command{ - Use: "activate [TARGET]", - Short: "Activate a layer and refresh sysext", - Long: `Activate a selected layer (symlink it to /var/lib/extensions) and refresh the system extensions store.`, + Use: "activate [TARGET...]", + Short: "Activate layers and refresh sysext", + Long: `Activate selected layers and refresh the system extensions store.`, RunE: activateCmd, + Args: cobra.MinimumNArgs(1), } -var ( - fFromFile *string -) +var fFromFile bool func init() { - fFromFile = ActivateCmd.Flags().StringP("file", "f", "", "Activate directly from file instead of cache") + ActivateCmd.Flags().BoolVarP(&fFromFile, "file", "f", false, "Parse positional arguments as files instead of layers") } func activateCmd(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return internal.NewPositionalError("TARGET") - } - - target_layer := args[0] - extensions_dir, err := filepath.Abs(path.Clean(internal.Config.ExtensionsDir)) if err != nil { return err } - if *fFromFile != "" { - if !strings.HasSuffix(target_layer, internal.ValidSysextExtension) { - target_layer, err = filepath.Abs(path.Clean(target_layer + internal.ValidSysextExtension)) + if fFromFile { + for _, target_file := range args { + if !strings.HasSuffix(target_file, internal.ValidSysextExtension) { + return errors.New("failed to parse file name, invalid sysext extension. Should be " + internal.ValidSysextExtension) + } + + deployment_path := path.Join(extensions_dir, path.Base(target_file)) + slog.Debug("Activating layer "+target_file, + slog.Bool("fromfile", fFromFile), + slog.String("file layer", target_file), + slog.String("path", deployment_path), + ) + + file_abs, err := filepath.Abs(path.Clean(target_file)) if err != nil { return err } - } - target_path := path.Join(extensions_dir, target_layer) - slog.Debug("acitavate", - slog.String("fromfile", *fFromFile), - slog.String("layer", target_layer), - slog.String("path", target_path), - ) - if err := os.Symlink(target_layer, path.Join(extensions_dir, target_layer)); err != nil { - return err + _ = os.Remove(file_abs) + + if err := os.Symlink(file_abs, path.Join(extensions_dir, path.Base(file_abs))); err != nil { + return err + } + slog.Info("Successfully activated layer " + path.Base(file_abs)) } - slog.Info(fmt.Sprintf("Successfully activated layer %s\n", path.Base(target_layer))) + return nil } @@ -65,25 +66,31 @@ func activateCmd(cmd *cobra.Command, args []string) error { return err } - current_blob_path := path.Join(cache_dir, target_layer, internal.CurrentBlobName) - if _, err := os.Stat(current_blob_path); err != nil { - return err - } + for _, target_layer := range args { + current_blob_path := path.Join(cache_dir, target_layer, internal.CurrentBlobName) + if _, err := os.Stat(current_blob_path); err != nil { + return errors.New("target layer " + target_layer + " could not be found") + } - if err := os.MkdirAll(internal.Config.ExtensionsDir, 0755); err != nil { - return err - } + if err := os.MkdirAll(internal.Config.ExtensionsDir, 0755); err != nil { + return err + } - target_path := path.Join(extensions_dir, path.Base(path.Dir(current_blob_path))+internal.ValidSysextExtension) - slog.Debug("acitavate", - slog.String("fromfile", *fFromFile), - slog.String("layer", target_layer), - slog.String("blob", current_blob_path), - ) - if err := os.Symlink(current_blob_path, target_path); err != nil { - return err + target_path := path.Join(extensions_dir, path.Base(path.Dir(current_blob_path))+internal.ValidSysextExtension) + slog.Debug("Activating layer", + slog.Bool("fromfile", fFromFile), + slog.String("layer", target_layer), + slog.String("blob", current_blob_path), + ) + + _ = os.Remove(target_path) + + if err := os.Symlink(current_blob_path, target_path); err != nil { + return err + } + + slog.Info("Successfully activated layer " + path.Base(target_layer)) } - slog.Info(fmt.Sprintf("Successfully activated layer %s\n", path.Base(target_layer))) return nil } From 0b5baf4c9a82c2b290601de9005768af34b771a4 Mon Sep 17 00:00:00 2001 From: Tulili Date: Mon, 26 Feb 2024 10:47:39 -0300 Subject: [PATCH 3/7] feat(activate): use concurrency when activating layers --- cmd/layer/activate/activate.go | 51 ++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/cmd/layer/activate/activate.go b/cmd/layer/activate/activate.go index 0d656ae..367f2ea 100644 --- a/cmd/layer/activate/activate.go +++ b/cmd/layer/activate/activate.go @@ -7,6 +7,7 @@ import ( "path" "path/filepath" "strings" + "sync" "github.com/spf13/cobra" "github.com/ublue-os/bext/internal" @@ -26,6 +27,17 @@ func init() { ActivateCmd.Flags().BoolVarP(&fFromFile, "file", "f", false, "Parse positional arguments as files instead of layers") } +func mapVal[T, U any](data []T, f func(T) U) []U { + + res := make([]U, 0, len(data)) + + for _, e := range data { + res = append(res, f(e)) + } + + return res +} + func activateCmd(cmd *cobra.Command, args []string) error { extensions_dir, err := filepath.Abs(path.Clean(internal.Config.ExtensionsDir)) if err != nil { @@ -33,6 +45,7 @@ func activateCmd(cmd *cobra.Command, args []string) error { } if fFromFile { + var wg sync.WaitGroup for _, target_file := range args { if !strings.HasSuffix(target_file, internal.ValidSysextExtension) { return errors.New("failed to parse file name, invalid sysext extension. Should be " + internal.ValidSysextExtension) @@ -50,14 +63,16 @@ func activateCmd(cmd *cobra.Command, args []string) error { return err } - _ = os.Remove(file_abs) - - if err := os.Symlink(file_abs, path.Join(extensions_dir, path.Base(file_abs))); err != nil { - return err - } - slog.Info("Successfully activated layer " + path.Base(file_abs)) + wg.Add(1) + go func() { + defer wg.Done() + _ = os.Remove(file_abs) + _ = os.Symlink(file_abs, path.Join(extensions_dir, path.Base(file_abs))) + }() } + wg.Done() + slog.Info("Successfully activated layers " + strings.Join(mapVal(args, path.Base), " ")) return nil } @@ -66,16 +81,17 @@ func activateCmd(cmd *cobra.Command, args []string) error { return err } + if err := os.MkdirAll(internal.Config.ExtensionsDir, 0755); err != nil { + return err + } + + var wg sync.WaitGroup for _, target_layer := range args { current_blob_path := path.Join(cache_dir, target_layer, internal.CurrentBlobName) if _, err := os.Stat(current_blob_path); err != nil { return errors.New("target layer " + target_layer + " could not be found") } - if err := os.MkdirAll(internal.Config.ExtensionsDir, 0755); err != nil { - return err - } - target_path := path.Join(extensions_dir, path.Base(path.Dir(current_blob_path))+internal.ValidSysextExtension) slog.Debug("Activating layer", slog.Bool("fromfile", fFromFile), @@ -83,14 +99,15 @@ func activateCmd(cmd *cobra.Command, args []string) error { slog.String("blob", current_blob_path), ) - _ = os.Remove(target_path) - - if err := os.Symlink(current_blob_path, target_path); err != nil { - return err - } - - slog.Info("Successfully activated layer " + path.Base(target_layer)) + wg.Add(1) + go func() { + defer wg.Done() + _ = os.Remove(target_path) + _ = os.Symlink(current_blob_path, target_path) + }() } + wg.Wait() + slog.Info("Successfully activated layers " + strings.Join(mapVal(args, path.Base), " ")) return nil } From 8f1915c5634265cb03c33add8fbc2a6e7d70bec7 Mon Sep 17 00:00:00 2001 From: Tulili Date: Mon, 26 Feb 2024 11:53:37 -0300 Subject: [PATCH 4/7] feat(add): concurrent adding + less ops for checking files --- cmd/layer/activate/activate.go | 15 +- cmd/layer/add/add.go | 287 ++++++++++++++++++--------------- internal/util.go | 12 ++ pkg/filecomp/checksum.go | 17 ++ 4 files changed, 186 insertions(+), 145 deletions(-) create mode 100644 internal/util.go diff --git a/cmd/layer/activate/activate.go b/cmd/layer/activate/activate.go index 367f2ea..7c61326 100644 --- a/cmd/layer/activate/activate.go +++ b/cmd/layer/activate/activate.go @@ -27,17 +27,6 @@ func init() { ActivateCmd.Flags().BoolVarP(&fFromFile, "file", "f", false, "Parse positional arguments as files instead of layers") } -func mapVal[T, U any](data []T, f func(T) U) []U { - - res := make([]U, 0, len(data)) - - for _, e := range data { - res = append(res, f(e)) - } - - return res -} - func activateCmd(cmd *cobra.Command, args []string) error { extensions_dir, err := filepath.Abs(path.Clean(internal.Config.ExtensionsDir)) if err != nil { @@ -72,7 +61,7 @@ func activateCmd(cmd *cobra.Command, args []string) error { } wg.Done() - slog.Info("Successfully activated layers " + strings.Join(mapVal(args, path.Base), " ")) + slog.Info("Successfully activated layers " + strings.Join(internal.MapVal(args, path.Base), " ")) return nil } @@ -108,6 +97,6 @@ func activateCmd(cmd *cobra.Command, args []string) error { } wg.Wait() - slog.Info("Successfully activated layers " + strings.Join(mapVal(args, path.Base), " ")) + slog.Info("Successfully activated layers " + strings.Join(internal.MapVal(args, path.Base), " ")) return nil } diff --git a/cmd/layer/add/add.go b/cmd/layer/add/add.go index 5b8c9b6..d633300 100644 --- a/cmd/layer/add/add.go +++ b/cmd/layer/add/add.go @@ -2,178 +2,201 @@ package add import ( "crypto/md5" - "crypto/sha256" "encoding/hex" "errors" + "fmt" "log/slog" "os" "path" "path/filepath" "strings" + "sync" "github.com/jedib0t/go-pretty/v6/progress" "github.com/spf13/cobra" "github.com/ublue-os/bext/internal" "github.com/ublue-os/bext/pkg/filecomp" "github.com/ublue-os/bext/pkg/fileio" + "github.com/ublue-os/bext/pkg/logging" - "github.com/ublue-os/bext/pkg/percentmanager" + percent "github.com/ublue-os/bext/pkg/percentmanager" ) var AddCmd = &cobra.Command{ - Use: "add [TARGET]", + Use: "add [TARGET...]", Short: "Add a built layer onto the cache and activate it", Long: `Copy TARGET over to cache-dir as a blob with the TARGET's sha256 as the filename`, RunE: addCmd, + Args: cobra.MinimumNArgs(1), } var ( - fNoSymlink *bool - fNoChecksum *bool - fOverride *bool - fLayerName *string + fNoSymlink bool + fNoChecksum bool + fOverride bool ) func init() { - fNoSymlink = AddCmd.Flags().Bool("no-symlink", false, "Do not activate layer once added to cache") - fNoChecksum = AddCmd.Flags().Bool("no-checksum", false, "Do not check if layer was properly added to cache") - fOverride = AddCmd.Flags().Bool("override", false, "Override blob if they are already written to cache") - fLayerName = AddCmd.Flags().String("layer-name", "", "Name of the layer that will be added onto") + AddCmd.Flags().BoolVar(&fNoSymlink, "no-symlink", false, "Do not activate layer once added to cache") + AddCmd.Flags().BoolVar(&fNoChecksum, "no-checksum", false, "Do not check if layer was properly added to cache") + AddCmd.Flags().BoolVar(&fOverride, "override", false, "Override blob if they are already written to cache") } -func addCmd(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return internal.NewPositionalError("TARGET") - } - target_layer := &internal.TargetLayerInfo{} - target_layer.Path = path.Clean(args[0]) - - var err error - target_layer.FileInfo, err = os.Stat(target_layer.Path) +func CheckBlobIntegrity(expectedSum []byte, target string) (bool, error) { + var written_file *os.File + written_file, err := os.Open(target) if err != nil { - return err + return false, err } + defer written_file.Close() + + return filecomp.CheckExpectedSum(md5.New(), expectedSum, written_file) +} + +func addCmd(cmd *cobra.Command, args []string) error { pw := percent.NewProgressWriter() if !*internal.Config.NoProgress { go pw.Render() slog.SetDefault(logging.NewMuteLogger()) } - var expectedSections int = 4 - - if !*fNoSymlink { - expectedSections++ - } - if !*fNoChecksum { - expectedSections += 2 - } - - add_tracker := percent.NewIncrementTracker(&progress.Tracker{Message: "Adding layer " + path.Base(target_layer.Path) + " to cache", Total: int64(100), Units: progress.UnitsDefault}, expectedSections) - pw.AppendTracker(add_tracker.Tracker) + pw.SetNumTrackersExpected(len(args)) - add_tracker.IncrementSection() if err := os.MkdirAll(internal.Config.CacheDir, 0755); err != nil { return err } - add_tracker.IncrementSection() - layer_sha := sha256.New() - layer_sha.Write(target_layer.Data) - target_layer.UUID = layer_sha.Sum(nil) - if err != nil { - return err - } - - if *fLayerName != "" { - slog.Warn("The path inside /usr/lib/sysext/extensions-* must be the same as the layer's name in order for it to function, please check if this is actually the case") - target_layer.LayerName = *fLayerName - } else { - target_layer.LayerName = strings.Split(path.Base(target_layer.Path), ".")[0] + var wg sync.WaitGroup + errChan := make(chan error, len(args)) + + for _, layer := range args { + wg.Add(1) + go func(layer string, errorChan chan<- error) { + defer wg.Done() + target_layer := &internal.TargetLayerInfo{} + target_layer.Path = path.Clean(layer) + + var err error + target_layer.FileInfo, err = os.Stat(target_layer.Path) + if err != nil { + errChan <- err + return + } + + var expectedSections int = 4 + + if !fNoSymlink { + expectedSections++ + } + if !fNoChecksum { + expectedSections += 2 + } + + add_tracker := percent.NewIncrementTracker(&progress.Tracker{Message: "Adding layer", Total: target_layer.FileInfo.Size(), Units: progress.UnitsBytes}, expectedSections) + pw.AppendTracker(add_tracker.Tracker) + + fileContent, err := os.ReadFile(target_layer.Path) + if err != nil { + errChan <- err + return + } + target_layer.Data = fileContent + target_layer.LayerName = strings.Split(path.Base(target_layer.Path), ".")[0] + layer_sha := md5.New() + layer_sha.Write(target_layer.Data) + target_layer.UUID = layer_sha.Sum(nil) + if err != nil { + errChan <- err + return + } + blob_filepath, err := filepath.Abs(path.Join(internal.Config.CacheDir, target_layer.LayerName, hex.EncodeToString(target_layer.UUID))) + if err != nil { + add_tracker.Tracker.MarkAsErrored() + errChan <- err + return + } + + add_tracker.IncrementSection() + if err := os.MkdirAll(path.Dir(blob_filepath), 0755); err != nil { + add_tracker.Tracker.MarkAsErrored() + errChan <- err + return + } + + if fileio.FileExist(blob_filepath) && !fOverride { + add_tracker.Tracker.MarkAsErrored() + errChan <- errors.New("Blob " + path.Base(blob_filepath) + " is already in cache") + return + } + + add_tracker.IncrementSection() + slog.Warn(fmt.Sprintf("Copying blob %s %s", target_layer.Path, blob_filepath)) + if err := fileio.FileCopy(target_layer.Path, blob_filepath); err != nil { + errChan <- err + return + } + + if !fNoChecksum { + add_tracker.Tracker.Message = "Checking blob" + + add_tracker.IncrementSection() + integrity, err := CheckBlobIntegrity(target_layer.UUID, blob_filepath) + + if err != nil || !integrity { + add_tracker.Tracker.MarkAsErrored() + errChan <- fmt.Errorf("copied blobs did not match. source: %s ; target: %s", target_layer.Path, blob_filepath) + return + } + } + + if fNoSymlink { + add_tracker.Tracker.MarkAsDone() + return + } + + var current_blob_path string + current_blob_path, err = filepath.Abs(path.Join(path.Dir(blob_filepath), internal.CurrentBlobName)) + if err != nil { + errChan <- err + return + } + slog.Debug("Refreshing symlink", slog.String("path", current_blob_path)) + add_tracker.IncrementSection() + if _, err := os.Lstat(current_blob_path); err == nil { + err = os.Remove(current_blob_path) + if err != nil { + add_tracker.Tracker.MarkAsErrored() + errChan <- err + return + } + } else if errors.Is(err, os.ErrNotExist) { + + } else { + add_tracker.Tracker.MarkAsErrored() + errChan <- err + return + } + + err = os.Symlink(blob_filepath, current_blob_path) + if err != nil { + add_tracker.Tracker.MarkAsErrored() + errChan <- err + return + } + add_tracker.Tracker.MarkAsDone() + }(layer, errChan) + } + + go func() { + wg.Wait() + close(errChan) + }() + + for err := range errChan { + slog.Warn(fmt.Sprintf("Error encountered when adding blobs: %s", err.Error()), slog.String("error", err.Error())) + } + + if len(errChan) == 0 { + slog.Info("Successfully added blobs to cache", slog.String("blobs", strings.Join(args, " "))) } - var blob_filepath string - blob_filepath, err = filepath.Abs(path.Join(internal.Config.CacheDir, target_layer.LayerName, hex.EncodeToString(target_layer.UUID))) - if err != nil { - add_tracker.Tracker.MarkAsErrored() - return err - } - - add_tracker.IncrementSection() - if err := os.MkdirAll(path.Dir(blob_filepath), 0755); err != nil { - add_tracker.Tracker.MarkAsErrored() - return err - } - - if fileio.FileExist(blob_filepath) && !*fOverride { - slog.Warn("Blob is already in cache") - add_tracker.Tracker.MarkAsErrored() - os.Exit(1) - } - - add_tracker.IncrementSection() - if err := fileio.FileCopy(target_layer.Path, blob_filepath); err != nil { - return err - } - - if !*fNoChecksum { - add_tracker.Tracker.Message = "Checking blob" - - add_tracker.IncrementSection() - var written_file *os.File - written_file, err = os.Open(blob_filepath) - if err != nil { - add_tracker.Tracker.MarkAsErrored() - return err - } - defer written_file.Close() - - var tlayer_fileobj *os.File - tlayer_fileobj, err = os.Open(target_layer.Path) - if err != nil { - return err - } - defer tlayer_fileobj.Close() - - add_tracker.IncrementSection() - _, err = filecomp.CheckFilesAreEqual(md5.New(), tlayer_fileobj, written_file) - if err != nil { - slog.Warn("Copied blobs did not match") - return err - } - } - - if *fNoSymlink { - slog.Info("Successfully added blob to cache", slog.String("blob_path", blob_filepath)) - add_tracker.Tracker.MarkAsDone() - return nil - } - - var current_blob_path string - current_blob_path, err = filepath.Abs(path.Join(path.Dir(blob_filepath), internal.CurrentBlobName)) - if err != nil { - return err - } - add_tracker.Tracker.Message = "Refreshing symlink" - slog.Debug("Refreshing symlink", slog.String("path", current_blob_path)) - add_tracker.IncrementSection() - if _, err := os.Lstat(current_blob_path); err == nil { - err = os.Remove(current_blob_path) - if err != nil { - add_tracker.Tracker.MarkAsErrored() - return err - } - } else if errors.Is(err, os.ErrNotExist) { - - } else { - add_tracker.Tracker.MarkAsErrored() - return err - } - - err = os.Symlink(blob_filepath, current_blob_path) - if err != nil { - add_tracker.Tracker.MarkAsErrored() - return err - } - add_tracker.Tracker.MarkAsDone() - - slog.Info("Successfully added blob to cache", slog.String("blob_path", blob_filepath)) return nil } diff --git a/internal/util.go b/internal/util.go new file mode 100644 index 0000000..695702e --- /dev/null +++ b/internal/util.go @@ -0,0 +1,12 @@ +package internal + +func MapVal[T, U any](data []T, f func(T) U) []U { + + res := make([]U, 0, len(data)) + + for _, e := range data { + res = append(res, f(e)) + } + + return res +} diff --git a/pkg/filecomp/checksum.go b/pkg/filecomp/checksum.go index 208ab7d..8909cca 100644 --- a/pkg/filecomp/checksum.go +++ b/pkg/filecomp/checksum.go @@ -32,6 +32,23 @@ func GetFileChecksum(file *os.File, hash hash.Hash) ([]byte, error) { return hash.Sum(nil), nil } +func CheckExpectedSum(hashing_algo hash.Hash, expectedSum []byte, files ...*os.File) (bool, error) { + for _, file := range files { + checksum, err := GetFileChecksum(file, hashing_algo) + if err != nil { + return false, err + } + + if !reflect.DeepEqual(checksum, expectedSum) { + return false, &ChecksumError{ + Message: "Could not verify that file is equal to checksum", + } + } + } + + return true, nil +} + func CheckFilesAreEqual(hashing_algo hash.Hash, files ...*os.File) (bool, error) { var last_file_sum []byte var err error From 26e313a367c193c2405d02bbf2b9a3dea12a7227 Mon Sep 17 00:00:00 2001 From: Tulili Date: Mon, 26 Feb 2024 17:19:36 -0300 Subject: [PATCH 5/7] feat(deactivate): concurrent deactivation of layers --- cmd/layer/deactivate/deactivate.go | 49 ++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/cmd/layer/deactivate/deactivate.go b/cmd/layer/deactivate/deactivate.go index 6d7203c..285e474 100644 --- a/cmd/layer/deactivate/deactivate.go +++ b/cmd/layer/deactivate/deactivate.go @@ -1,44 +1,61 @@ package deactivate import ( - "github.com/spf13/cobra" - "github.com/ublue-os/bext/internal" + "fmt" "log/slog" "os" "path" "path/filepath" + "strings" + "sync" + + "github.com/spf13/cobra" + "github.com/ublue-os/bext/internal" ) var DeactivateCmd = &cobra.Command{ - Use: "deactivate [TARGET]", + Use: "deactivate [TARGET...]", Short: "Deactivate a layer and refresh sysext", Long: `Deativate a selected layer (unsymlink it from /var/lib/extensions) and refresh the system extensions store.`, RunE: deactivateCmd, + Args: cobra.MinimumNArgs(1), } func deactivateCmd(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return internal.NewPositionalError("TARGET") - } - - target_layer := args[0] - extensions_dir, err := filepath.Abs(path.Clean(internal.Config.ExtensionsDir)) if err != nil { return err } - target_layer_path := path.Join(extensions_dir, target_layer+internal.ValidSysextExtension) - - if _, err := os.Stat(target_layer_path); err != nil { - return err + var ( + errChan chan error + wg sync.WaitGroup + ) + + for _, target_layer := range args { + wg.Add(1) + go func(errChan chan<- error, target string) { + defer wg.Done() + + if err := os.Remove(path.Join(extensions_dir, target+internal.ValidSysextExtension)); err != nil { + errChan <- err + return + } + }(errChan, target_layer) } - if err := os.Remove(target_layer_path); err != nil { - return err + go func() { + wg.Wait() + close(errChan) + }() + + for err := range errChan { + slog.Warn(fmt.Sprintf("Error encountered when deactivating layers: %s", err.Error()), slog.String("error", err.Error())) } - slog.Info("Successfully deactivated " + target_layer) + if len(errChan) == 0 { + slog.Info("Successfully deactivated layers", slog.String("layers", strings.Join(args, " "))) + } return nil } From f33f7efa0f764c221fb5eb06288134040ecec3b5 Mon Sep 17 00:00:00 2001 From: Tulili Date: Mon, 26 Feb 2024 19:13:47 -0300 Subject: [PATCH 6/7] fix(activate): minimize code for activate cmd + lenght for errChan in deactivate cmd --- cmd/layer/activate/activate.go | 111 ++++++++++++++++------------- cmd/layer/deactivate/deactivate.go | 2 +- 2 files changed, 62 insertions(+), 51 deletions(-) diff --git a/cmd/layer/activate/activate.go b/cmd/layer/activate/activate.go index 7c61326..a1147c5 100644 --- a/cmd/layer/activate/activate.go +++ b/cmd/layer/activate/activate.go @@ -2,6 +2,7 @@ package activate import ( "errors" + "fmt" "log/slog" "os" "path" @@ -21,10 +22,14 @@ var ActivateCmd = &cobra.Command{ Args: cobra.MinimumNArgs(1), } -var fFromFile bool +var ( + fFromFile bool + fOverride bool +) func init() { ActivateCmd.Flags().BoolVarP(&fFromFile, "file", "f", false, "Parse positional arguments as files instead of layers") + ActivateCmd.Flags().BoolVar(&fOverride, "override", true, "Write over old symlinks") } func activateCmd(cmd *cobra.Command, args []string) error { @@ -33,38 +38,10 @@ func activateCmd(cmd *cobra.Command, args []string) error { return err } - if fFromFile { - var wg sync.WaitGroup - for _, target_file := range args { - if !strings.HasSuffix(target_file, internal.ValidSysextExtension) { - return errors.New("failed to parse file name, invalid sysext extension. Should be " + internal.ValidSysextExtension) - } - - deployment_path := path.Join(extensions_dir, path.Base(target_file)) - slog.Debug("Activating layer "+target_file, - slog.Bool("fromfile", fFromFile), - slog.String("file layer", target_file), - slog.String("path", deployment_path), - ) - - file_abs, err := filepath.Abs(path.Clean(target_file)) - if err != nil { - return err - } - - wg.Add(1) - go func() { - defer wg.Done() - _ = os.Remove(file_abs) - _ = os.Symlink(file_abs, path.Join(extensions_dir, path.Base(file_abs))) - }() - } - wg.Done() - - slog.Info("Successfully activated layers " + strings.Join(internal.MapVal(args, path.Base), " ")) - return nil - } - + var ( + errChan = make(chan error, len(args)) + wg sync.WaitGroup + ) cache_dir, err := filepath.Abs(path.Clean(internal.Config.CacheDir)) if err != nil { return err @@ -74,29 +51,63 @@ func activateCmd(cmd *cobra.Command, args []string) error { return err } - var wg sync.WaitGroup - for _, target_layer := range args { - current_blob_path := path.Join(cache_dir, target_layer, internal.CurrentBlobName) - if _, err := os.Stat(current_blob_path); err != nil { - return errors.New("target layer " + target_layer + " could not be found") - } - - target_path := path.Join(extensions_dir, path.Base(path.Dir(current_blob_path))+internal.ValidSysextExtension) - slog.Debug("Activating layer", + for _, target_file := range args { + slog.Debug("Activating layer "+target_file, slog.Bool("fromfile", fFromFile), - slog.String("layer", target_layer), - slog.String("blob", current_blob_path), + slog.String("layer", target_file), ) wg.Add(1) - go func() { + go func(errChan chan<- error, target string) { defer wg.Done() - _ = os.Remove(target_path) - _ = os.Symlink(current_blob_path, target_path) - }() + var ( + deployment_path string + target_path string + ) + + if !strings.HasSuffix(target, internal.ValidSysextExtension) && fFromFile { + errChan <- errors.New("failed to parse file name, invalid sysext extension. should be " + internal.ValidSysextExtension) + return + } + + if fFromFile { + layer_name := strings.Split(path.Base(target), ".")[0] + target_path = path.Join(extensions_dir, layer_name) + deployment_path, err = filepath.Abs(layer_name) + if err != nil { + errChan <- err + return + } + } else { + deployment_path = path.Join(cache_dir, target, internal.CurrentBlobName) + if _, err := os.Stat(deployment_path); err != nil { + errChan <- errors.New("target layer " + target + " could not be found") + return + } + target_path = path.Join(extensions_dir, target+internal.ValidSysextExtension) + } + if fOverride { + _ = os.Remove(target_path) + } else { + errChan <- errors.New(target + " is already activated") + } + if err := os.Symlink(deployment_path, target_path); err != nil { + errChan <- err + } + }(errChan, target_file) } - wg.Wait() - slog.Info("Successfully activated layers " + strings.Join(internal.MapVal(args, path.Base), " ")) + go func() { + wg.Wait() + close(errChan) + }() + + for err := range errChan { + slog.Warn(fmt.Sprintf("Error encountered when activating layers: %s", err.Error()), slog.String("error", err.Error())) + } + + if len(errChan) == 0 { + slog.Info("Successfully activated layers", slog.String("layers", strings.Join(args, " "))) + } return nil } diff --git a/cmd/layer/deactivate/deactivate.go b/cmd/layer/deactivate/deactivate.go index 285e474..8bc358a 100644 --- a/cmd/layer/deactivate/deactivate.go +++ b/cmd/layer/deactivate/deactivate.go @@ -28,7 +28,7 @@ func deactivateCmd(cmd *cobra.Command, args []string) error { } var ( - errChan chan error + errChan = make(chan error, len(args)) wg sync.WaitGroup ) From ceb6c97d5362f4dc4064c51067d1adc221299e69 Mon Sep 17 00:00:00 2001 From: Tulili Date: Mon, 26 Feb 2024 19:44:48 -0300 Subject: [PATCH 7/7] feat(remove): concurrency when deleting hashes and layers from cache --- cmd/layer/remove/remove.go | 110 +++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 47 deletions(-) diff --git a/cmd/layer/remove/remove.go b/cmd/layer/remove/remove.go index 3b39a57..5eaceca 100644 --- a/cmd/layer/remove/remove.go +++ b/cmd/layer/remove/remove.go @@ -1,10 +1,14 @@ package remove import ( + "errors" + "fmt" "log/slog" "os" "path" "path/filepath" + "strings" + "sync" "github.com/jedib0t/go-pretty/v6/progress" "github.com/spf13/cobra" @@ -15,24 +19,24 @@ import ( ) var RemoveCmd = &cobra.Command{ - Use: "remove", + Use: "remove [TARGET...]", Short: "Remove a layer from your managed layers", Long: `Remove either an entire layer or a specific hash in cache for that layer`, RunE: removeCmd, + Args: cobra.MinimumNArgs(1), } var ( - fHash *string - fDryRun *bool + fHash []string + fDryRun bool ) func init() { - fHash = RemoveCmd.Flags().StringP("hash", "h", "", "Remove specific hash from storage") - fDryRun = RemoveCmd.Flags().Bool("dry-run", false, "Do not remove anything") + RemoveCmd.Flags().StringSliceVarP(&fHash, "hash", "h", []string{}, "Remove specific hash from storage") + RemoveCmd.Flags().BoolVar(&fDryRun, "dry-run", false, "Do not remove anything") } func removeCmd(cmd *cobra.Command, args []string) error { - // todo dryrun flag slog.Info("Ignoring dryrun flag", "dryrun", fDryRun) @@ -41,61 +45,73 @@ func removeCmd(cmd *cobra.Command, args []string) error { go pw.Render() slog.SetDefault(logging.NewMuteLogger()) } - - if len(args) < 1 { - return internal.NewPositionalError("TARGET") + if len(fHash) > 1 { + pw.SetNumTrackersExpected(len(fHash)) + } else { + pw.SetNumTrackersExpected(len(args)) } - - target_layer := args[0] - cache_dir, err := filepath.Abs(path.Clean(internal.Config.CacheDir)) if err != nil { return err } - var message string = "Deleting layer " + target_layer - var expectedSections = 4 + var ( + wg sync.WaitGroup + errChan = make(chan error, len(fHash)) + ) - if *fHash != "" { - message = "Deleting hash " + target_layer - expectedSections = expectedSections + 1 - } - delete_tracker := percent.NewIncrementTracker(&progress.Tracker{Message: message, Total: int64(100), Units: progress.UnitsDefault}, expectedSections) - pw.AppendTracker(delete_tracker.Tracker) - - if *fHash != "" { - delete_tracker.IncrementSection() - err := os.Remove(path.Join(cache_dir, target_layer, *fHash)) - if err != nil { - return err - } - slog.Info("Successfuly deleted " + *fHash) - return nil + if len(args) > 1 && len(fHash) > 1 { + return errors.New("when removing hashes, it is required to only specify one layer") } - delete_tracker.IncrementSection() - slog.Debug("Deleting layer", slog.String("target", target_layer)) - err = os.RemoveAll(path.Join(cache_dir, target_layer)) - if err != nil { - return err + for _, hash := range fHash { + wg.Add(1) + go func(errChan chan<- error, target string) { + defer wg.Done() + delete_tracker := percent.NewIncrementTracker(&progress.Tracker{Message: "Deleting hash", Total: int64(100), Units: progress.UnitsDefault}, 1) + defer delete_tracker.Tracker.MarkAsDone() + pw.AppendTracker(delete_tracker.Tracker) + + if len(fHash) > 0 { + err := os.Remove(path.Join(cache_dir, args[0], target)) + if err != nil { + errChan <- err + return + } + return + } else { + err := os.RemoveAll(path.Join(cache_dir, target)) + if err != nil { + errChan <- err + return + } + } + + deactivated_layer := path.Join(internal.Config.ExtensionsDir, target) + internal.ValidSysextExtension + if !fileio.FileExist(deactivated_layer) { + return + } + + err = os.Remove(deactivated_layer) + if err != nil { + errChan <- err + return + } + }(errChan, hash) } - delete_tracker.IncrementSection() - deactivated_layer := path.Join(internal.Config.ExtensionsDir, target_layer) + internal.ValidSysextExtension - if !fileio.FileExist(deactivated_layer) { - return nil - } + go func() { + wg.Wait() + close(errChan) + }() - delete_tracker.IncrementSection() - slog.Debug("Deactivating layer", slog.String("target", deactivated_layer)) - err = os.Remove(deactivated_layer) - if err != nil { - return err + for err := range errChan { + slog.Warn(fmt.Sprintf("Error encountered when deleting targets: %s", err.Error()), slog.String("error", err.Error())) } - slog.Info("Successfuly deleted "+deactivated_layer, slog.String("target", deactivated_layer)) + if len(errChan) == 0 { + slog.Info("Successfully deleted target from cache", slog.String("hashes", strings.Join(fHash, " "))) + } - delete_tracker.IncrementSection() - delete_tracker.Tracker.MarkAsDone() return nil }