From 649ff653b318cfd519be65033302188614246bfa Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Mon, 16 Dec 2024 17:54:05 +0100 Subject: [PATCH 1/4] test: Remove image just after upload to registry Remove an image from a local docker store just after its been uploaded to a registry. It makes the initial test environment clean, so there are no any images in a local docker store before the tests start running. Signed-off-by: Mike Sul --- test/fixtures/preload-images.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/fixtures/preload-images.sh b/test/fixtures/preload-images.sh index 2c28d3f..5180768 100755 --- a/test/fixtures/preload-images.sh +++ b/test/fixtures/preload-images.sh @@ -24,6 +24,8 @@ if ! check_image; then docker pull ${SRC_IMAGE} docker tag ${SRC_IMAGE} ${IMAGE_URI} docker push ${IMAGE_URI} + docker image rm ${SRC_IMAGE} + docker image rm ${IMAGE_URI} else echo "Image ${IMAGE_URI} exists in the registry." fi From 2013490265b45c6ed0880f5e38a4fccb29484728 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Mon, 16 Dec 2024 18:22:28 +0100 Subject: [PATCH 2/4] test: Make app publishing self-contained Pull app images before publishing app, so the publish procedure can gather info about each app image layers. Also, removes the pulled images after app been published, so the following tests emulate a device environment in which app images are not pre-pulled. Signed-off-by: Mike Sul --- test/fixtures/composectl_cmds.go | 54 +++++++++++++++++++++++++++----- test/integration/smoke_test.go | 4 +-- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/test/fixtures/composectl_cmds.go b/test/fixtures/composectl_cmds.go index 558d791..2086683 100644 --- a/test/fixtures/composectl_cmds.go +++ b/test/fixtures/composectl_cmds.go @@ -6,10 +6,12 @@ import ( "encoding/json" "fmt" composectl "github.com/foundriesio/composeapp/cmd/composectl/cmd" + "gopkg.in/yaml.v3" rand2 "math/rand" "os" "os/exec" "path" + "path/filepath" "testing" "time" ) @@ -27,7 +29,7 @@ type ( PublishOpts struct { PublishLayersManifest bool - LayersMetaFile string + PublishLayersMetaFile bool Registry string } ) @@ -54,9 +56,9 @@ func WithLayersManifest(addLayerManifest bool) func(opts *PublishOpts) { opts.PublishLayersManifest = addLayerManifest } } -func WithLayersMeta(layersMetaFile string) func(opts *PublishOpts) { +func WithLayersMeta(layersMetaFile bool) func(opts *PublishOpts) { return func(opts *PublishOpts) { - opts.LayersMetaFile = layersMetaFile + opts.PublishLayersMetaFile = layersMetaFile } } @@ -73,8 +75,44 @@ func NewApp(t *testing.T, composeDef string, name ...string) *App { return app } +func (a *App) pullImages(t *testing.T) error { + b, err := os.ReadFile(path.Join(a.Dir, "docker-compose.yml")) + check(t, err) + var composeProj map[string]interface{} + check(t, yaml.Unmarshal(b, &composeProj)) + services := composeProj["services"] + for _, v := range services.(map[string]interface{}) { + image := v.(map[string]interface{})["image"] + c := exec.Command("docker", "pull", image.(string)) + output, cmdErr := c.CombinedOutput() + checkf(t, cmdErr, "failed to pull app images: %s\n", output) + } + return err +} + +func (a *App) removeImages(t *testing.T) error { + b, err := os.ReadFile(path.Join(a.Dir, "docker-compose.yml")) + check(t, err) + var composeProj map[string]interface{} + check(t, yaml.Unmarshal(b, &composeProj)) + services := composeProj["services"] + removedImages := map[string]bool{} + for _, v := range services.(map[string]interface{}) { + image := v.(map[string]interface{})["image"] + if _, ok := removedImages[image.(string)]; ok { + continue + } + c := exec.Command("docker", "image", "rm", image.(string)) + output, cmdErr := c.CombinedOutput() + checkf(t, cmdErr, "failed to pull app images: %s\n", output) + removedImages[image.(string)] = true + } + return err +} + func (a *App) Publish(t *testing.T, publishOpts ...func(*PublishOpts)) { - opts := PublishOpts{PublishLayersManifest: true} + check(t, a.pullImages(t)) + opts := PublishOpts{PublishLayersManifest: true, PublishLayersMetaFile: true} for _, o := range publishOpts { o(&opts) } @@ -93,8 +131,10 @@ func (a *App) Publish(t *testing.T, publishOpts ...func(*PublishOpts)) { if !opts.PublishLayersManifest { args = append(args, "--layers-manifest=false") } - if len(opts.LayersMetaFile) > 0 { - args = append(args, "--layers-meta", opts.LayersMetaFile) + if opts.PublishLayersMetaFile { + layersMetaFile := GenerateLayersMetaFile(t, filepath.Dir(a.Dir)) + defer os.RemoveAll(layersMetaFile) + args = append(args, "--layers-meta", layersMetaFile) } runCmd(t, a.Dir, args...) b, err := os.ReadFile(digestFile) @@ -102,6 +142,7 @@ func (a *App) Publish(t *testing.T, publishOpts ...func(*PublishOpts)) { a.PublishedUri = baseUri + "@" + string(b) fmt.Printf("published app uri: %s\n", a.PublishedUri) }) + check(t, a.removeImages(t)) } func (a *App) Pull(t *testing.T) { @@ -166,7 +207,6 @@ func (a *App) CheckInstalled(t *testing.T) { output := runCmd(t, a.Dir, "check", "--local", "--install", a.PublishedUri, "--format", "json") checkResult := composectl.CheckAndInstallResult{} check(t, json.Unmarshal(output, &checkResult)) - if len(checkResult.FetchCheck.MissingBlobs) > 0 { t.Errorf("there are missing app blobs: %+v\n", checkResult.FetchCheck.MissingBlobs) } diff --git a/test/integration/smoke_test.go b/test/integration/smoke_test.go index 6d99548..8564d1c 100644 --- a/test/integration/smoke_test.go +++ b/test/integration/smoke_test.go @@ -2,7 +2,6 @@ package e2e_tests import ( f "github.com/foundriesio/composeapp/test/fixtures" - "path/filepath" "testing" ) @@ -14,10 +13,9 @@ services: command: sh -c "while true; do sleep 60; done" ` app := f.NewApp(t, appComposeDef) - layersMetaFile := f.GenerateLayersMetaFile(t, filepath.Dir(app.Dir)) smokeTest := func(registry string, layersManifest bool) { - app.Publish(t, f.WithRegistry(registry), f.WithLayersManifest(layersManifest), f.WithLayersMeta(layersMetaFile)) + app.Publish(t, f.WithRegistry(registry), f.WithLayersManifest(layersManifest)) app.Pull(t) defer app.Remove(t) From 8423bbc051db5f872d8fef5e69eeb03302375901 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Tue, 17 Dec 2024 10:20:36 +0100 Subject: [PATCH 3/4] uninstall: Add flag to optionally prune images Add a new flag to enable/disable app images pruning in the docker store just after removing an app installation from the compose/project directory. By default the flag is set to false (no pruning). If set to true, then app images that are not used by any containers (regardless of a container state) are removed from the docker store. Signed-off-by: Mike Sul --- cmd/composectl/cmd/uninstall.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/composectl/cmd/uninstall.go b/cmd/composectl/cmd/uninstall.go index b6125ef..7f965d9 100644 --- a/cmd/composectl/cmd/uninstall.go +++ b/cmd/composectl/cmd/uninstall.go @@ -2,6 +2,8 @@ package composectl import ( "fmt" + "github.com/docker/docker/api/types/filters" + "github.com/foundriesio/composeapp/pkg/compose" "github.com/spf13/cobra" "os" "path/filepath" @@ -17,6 +19,7 @@ var uninstallCmd = &cobra.Command{ type ( uninstallOptions struct { ignoreNonInstalled bool + prune bool } ) @@ -24,6 +27,7 @@ func init() { opts := uninstallOptions{} uninstallCmd.Flags().BoolVar(&opts.ignoreNonInstalled, "ignore-non-installed", false, "Do not yield error if app installation is not found") + uninstallCmd.Flags().BoolVar(&opts.prune, "prune", false, "prune unused images in the docker store") uninstallCmd.Run = func(cmd *cobra.Command, args []string) { uninstallApps(cmd, args, &opts) } @@ -44,4 +48,10 @@ func uninstallApps(cmd *cobra.Command, args []string, opts *uninstallOptions) { } DieNotNil(os.RemoveAll(appComposeDir)) } + if opts.prune { + cli, err := compose.GetDockerClient(dockerHost) + DieNotNil(err) + _, err = cli.ImagesPrune(cmd.Context(), filters.NewArgs(filters.Arg("dangling", "false"))) + DieNotNil(err) + } } From 51b6da364c54a5451c55a0043f0344c1bc9ca52a Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Tue, 17 Dec 2024 10:44:35 +0100 Subject: [PATCH 4/4] test: Prune app images when uninstalling app Signed-off-by: Mike Sul --- test/fixtures/composectl_cmds.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/composectl_cmds.go b/test/fixtures/composectl_cmds.go index 2086683..278a0b5 100644 --- a/test/fixtures/composectl_cmds.go +++ b/test/fixtures/composectl_cmds.go @@ -158,7 +158,7 @@ func (a *App) Install(t *testing.T) { } func (a *App) Uninstall(t *testing.T) { - a.runCmd(t, "uninstall app", "uninstall", a.Name) + a.runCmd(t, "uninstall app", "uninstall", "--prune=true", a.Name) } func (a *App) Run(t *testing.T) {