Merge branch 'main' into eric/fix-lint-upgrade

core/provider/digitalocean/task.go

@@ -325,19 +325,36 @@ func (p *Provider) RunCommand(ctx context.Context, taskName string, command []st
 
 	defer resp.Close()
 
-	execInspect, err := dockerClient.ContainerExecInspect(ctx, exec.ID)
+	lastExitCode := 0
+
+	err = util.WaitForCondition(ctx, 10*time.Second, 100*time.Millisecond, func() (bool, error) {
+		execInspect, err := dockerClient.ContainerExecInspect(ctx, exec.ID)
+		if err != nil {
+			return false, err
+		}
+
+		if execInspect.Running {
+			return false, nil
+		}
+
+		lastExitCode = execInspect.ExitCode
+
+		return true, nil
+	})
+
 	if err != nil {
-		return "", "", 0, err
+		p.logger.Error("failed to wait for exec", zap.Error(err), zap.String("taskName", taskName))
+		return "", "", lastExitCode, err
 	}
 
 	var stdout, stderr bytes.Buffer
 
 	_, err = stdcopy.StdCopy(&stdout, &stderr, resp.Reader)
 	if err != nil {
-		return "", "", 0, err
+		return "", "", lastExitCode, err
 	}
 
-	return stdout.String(), stderr.String(), execInspect.ExitCode, nil
+	return stdout.String(), stderr.String(), lastExitCode, nil
 }
 
 func (p *Provider) RunCommandWhileStopped(ctx context.Context, taskName string, definition provider.TaskDefinition, command []string) (string, string, int, error) {
@@ -381,8 +398,16 @@ func (p *Provider) RunCommandWhileStopped(ctx context.Context, taskName string,
 		return "", "", 0, err
 	}
 
-	// nolint
-	defer dockerClient.ContainerRemove(ctx, createdContainer.ID, container.RemoveOptions{Force: true})
+	defer func() {
+		if _, err := dockerClient.ContainerInspect(ctx, createdContainer.ID); err != nil && dockerclient.IsErrNotFound(err) {
+			// auto-removed, but not detected as autoremoved
+			return
+		}
+
+		if err := dockerClient.ContainerRemove(ctx, createdContainer.ID, container.RemoveOptions{Force: true}); err != nil {
+			p.logger.Error("failed to remove container", zap.Error(err), zap.String("taskName", taskName), zap.String("id", createdContainer.ID))
+		}
+	}()
 
 	if err := startContainerWithBlock(ctx, dockerClient, createdContainer.ID); err != nil {
 		p.logger.Error("failed to start container", zap.Error(err), zap.String("taskName", taskName))
@@ -408,10 +433,26 @@ func (p *Provider) RunCommandWhileStopped(ctx context.Context, taskName string,
 
 	defer resp.Close()
 
-	execInspect, err := dockerClient.ContainerExecInspect(ctx, exec.ID)
+	lastExitCode := 0
+
+	err = util.WaitForCondition(ctx, 10*time.Second, 100*time.Millisecond, func() (bool, error) {
+		execInspect, err := dockerClient.ContainerExecInspect(ctx, exec.ID)
+		if err != nil {
+			return false, err
+		}
+
+		if execInspect.Running {
+			return false, nil
+		}
+
+		lastExitCode = execInspect.ExitCode
+
+		return true, nil
+	})
+
 	if err != nil {
-		p.logger.Error("failed to inspect exec", zap.Error(err), zap.String("taskName", taskName))
-		return "", "", 0, err
+		p.logger.Error("failed to wait for exec", zap.Error(err), zap.String("taskName", taskName))
+		return "", "", lastExitCode, err
 	}
 
 	var stdout, stderr bytes.Buffer
@@ -420,7 +461,7 @@ func (p *Provider) RunCommandWhileStopped(ctx context.Context, taskName string,
 		return "", "", 0, err
 	}
 
-	return stdout.String(), stderr.String(), execInspect.ExitCode, err
+	return stdout.String(), stderr.String(), lastExitCode, err
 }
 
 func startContainerWithBlock(ctx context.Context, dockerClient *dockerclient.Client, containerID string) error {

core/provider/docker/task.go

@@ -235,18 +235,35 @@ func (p *Provider) RunCommand(ctx context.Context, id string, command []string)
 
 	defer resp.Close()
 
-	execInspect, err := p.dockerClient.ContainerExecInspect(ctx, exec.ID)
+	lastExitCode := 0
+
+	err = util.WaitForCondition(ctx, 10*time.Second, 100*time.Millisecond, func() (bool, error) {
+		execInspect, err := p.dockerClient.ContainerExecInspect(ctx, exec.ID)
+		if err != nil {
+			return false, err
+		}
+
+		if execInspect.Running {
+			return false, nil
+		}
+
+		lastExitCode = execInspect.ExitCode
+
+		return true, nil
+	})
+
 	if err != nil {
-		return "", "", 0, err
+		p.logger.Error("failed to wait for exec", zap.Error(err), zap.String("id", id))
+		return "", "", lastExitCode, err
 	}
 
 	var stdout, stderr bytes.Buffer
 	_, err = stdcopy.StdCopy(&stdout, &stderr, resp.Reader)
 	if err != nil {
-		return "", "", 0, err
+		return "", "", lastExitCode, err
 	}
 
-	return stdout.String(), stderr.String(), execInspect.ExitCode, nil
+	return stdout.String(), stderr.String(), lastExitCode, nil
 }
 
 func (p *Provider) RunCommandWhileStopped(ctx context.Context, id string, definition provider.TaskDefinition, command []string) (string, string, int, error) {

core/provider/docker/volume.go

@@ -16,6 +16,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/docker/client"
 
 	"github.com/skip-mev/petri/core/v2/provider"
 )
@@ -123,11 +124,15 @@ func (p *Provider) WriteFile(ctx context.Context, id, relPath string, content []
 			return
 		}
 
-		// nolint // will fix later
+		if _, err := p.dockerClient.ContainerInspect(ctx, cc.ID); err != nil && client.IsErrNotFound(err) {
+			// auto-removed, but not detected as autoremoved
+			return
+		}
+
 		if err := p.dockerClient.ContainerRemove(ctx, cc.ID, container.RemoveOptions{
 			Force: true,
 		}); err != nil {
-			// TODO fix logging
+			logger.Error("failed to remove writefile container", zap.String("id", cc.ID), zap.Error(err))
 		}
 	}()
 
@@ -241,11 +246,15 @@ func (p *Provider) ReadFile(ctx context.Context, id, relPath string) ([]byte, er
 	logger.Debug("created getfile container", zap.String("id", cc.ID))
 
 	defer func() {
+		if _, err := p.dockerClient.ContainerInspect(ctx, cc.ID); err != nil && client.IsErrNotFound(err) {
+			// auto-removed, but not detected as autoremoved
+			return
+		}
+
 		if err := p.dockerClient.ContainerRemove(ctx, cc.ID, container.RemoveOptions{
 			Force: true,
 		}); err != nil {
 			logger.Error("failed cleaning up the getfile container", zap.Error(err))
-			// todo fix logging
 		}
 	}()
 
@@ -327,7 +336,10 @@ func (p *Provider) DownloadDir(ctx context.Context, id, relPath, localPath strin
 	}
 
 	defer func() {
-		// nolint // will fix later
+		if _, err := p.dockerClient.ContainerInspect(ctx, cc.ID); err != nil && client.IsErrNotFound(err) {
+			return
+		}
+
 		if err := p.dockerClient.ContainerRemove(ctx, cc.ID, container.RemoveOptions{
 			Force: true,
 		}); err != nil {
@@ -407,8 +419,8 @@ func (p *Provider) SetVolumeOwner(ctx context.Context, volumeName, uid, gid stri
 			User: "0",
 		},
 		&container.HostConfig{
-			Binds: []string{volumeName + ":" + mountPath},
-			// AutoRemove: true,
+			Binds:      []string{volumeName + ":" + mountPath},
+			AutoRemove: true,
 		},
 		nil, // No networking necessary.
 		nil,
@@ -423,8 +435,12 @@ func (p *Provider) SetVolumeOwner(ctx context.Context, volumeName, uid, gid stri
 			// No need to attempt removing the container if we successfully started and waited for it to complete.
 			return
 		}
+
+		if _, err := p.dockerClient.ContainerInspect(ctx, cc.ID); err != nil && client.IsErrNotFound(err) {
+			// auto-removed, but not detected as autoremoved
+			return
+		}
 
-		// nolint // will fix later
 		if err := p.dockerClient.ContainerRemove(ctx, cc.ID, container.RemoveOptions{
 			Force: true,
 		}); err != nil {

cosmos/node/genesis.go

@@ -75,7 +75,11 @@ func (n *Node) AddGenesisAccount(ctx context.Context, address string, genesisAmo
 	n.logger.Debug("add-genesis-account", zap.String("stdout", stdout), zap.String("stderr", stderr), zap.Int("exitCode", exitCode))
 
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to add genesis account: %w", err)
+	}
+
+	if exitCode != 0 {
+		return fmt.Errorf("failed to add genesis account (exitcode=%d): %s", exitCode, stderr)
 	}
 
 	return nil
@@ -102,11 +106,15 @@ func (n *Node) GenerateGenTx(ctx context.Context, genesisSelfDelegation types.Co
 	stdout, stderr, exitCode, err := n.Task.RunCommand(ctx, command)
 	n.logger.Debug("gentx", zap.String("stdout", stdout), zap.String("stderr", stderr), zap.Int("exitCode", exitCode))
 
+	if err != nil {
+		return fmt.Errorf("failed to generate genesis transaction: %w", err)
+	}
+
 	if exitCode != 0 {
-		return fmt.Errorf("failed to generate genesis transaction: %s (exitcode=%d)", stderr, exitCode)
+		return fmt.Errorf("failed to generate genesis transaction (exitcode=%d): %s", exitCode, stderr)
 	}
 
-	return err
+	return nil
 }
 
 // CollectGenTxs collects the genesis transactions from the node and create a finalized genesis file
@@ -124,7 +132,15 @@ func (n *Node) CollectGenTxs(ctx context.Context) error {
 	stdout, stderr, exitCode, err := n.Task.RunCommand(ctx, n.BinCommand(command...))
 	n.logger.Debug("collect-gentxs", zap.String("stdout", stdout), zap.String("stderr", stderr), zap.Int("exitCode", exitCode))
 
-	return err
+	if err != nil {
+		return fmt.Errorf("failed to collect genesis transactions: %w", err)
+	}
+
+	if exitCode != 0 {
+		return fmt.Errorf("failed to collect genesis transactions (exitcode=%d): %s", exitCode, stderr)
+	}
+
+	return nil
 }
 
 // OverwriteGenesisFile overwrites the genesis file on the node with the provided genesis file

cosmos/node/init.go

@@ -2,6 +2,7 @@ package node
 
 import (
 	"context"
+	"fmt"
 
 	"go.uber.org/zap"
 )
@@ -14,5 +15,13 @@ func (n *Node) InitHome(ctx context.Context) error {
 	stdout, stderr, exitCode, err := n.Task.RunCommand(ctx, n.BinCommand([]string{"init", n.Definition.Name, "--chain-id", chainConfig.ChainId}...))
 	n.logger.Debug("init home", zap.String("stdout", stdout), zap.String("stderr", stderr), zap.Int("exitCode", exitCode))
 
-	return err
+	if err != nil {
+		return fmt.Errorf("failed to init home: %w", err)
+	}
+
+	if exitCode != 0 {
+		return fmt.Errorf("failed to init home (exit code %d): %s", exitCode, stderr)
+	}
+
+	return nil
 }

cosmos/node/keys.go

@@ -58,12 +58,16 @@ func (n *Node) KeyBech32(ctx context.Context, name, bech string) (string, error)
 		command = append(command, "--bech", bech)
 	}
 
-	stdout, stderr, _, err := n.Task.RunCommand(ctx, command)
+	stdout, stderr, exitCode, err := n.Task.RunCommand(ctx, command)
 	n.logger.Debug("show key", zap.String("name", name), zap.String("stdout", stdout), zap.String("stderr", stderr))
 
 	if err != nil {
 		return "", fmt.Errorf("failed to show key %q (stderr=%q): %w", name, stderr, err)
 	}
 
+	if exitCode != 0 {
+		return "", fmt.Errorf("failed to show key %q (exitcode=%d): %s", name, exitCode, stderr)
+	}
+
 	return util.CleanDockerOutput(stdout), nil
 }