Skip to content

Commit

Permalink
Merge pull request #20050 from umohnani8/farm-build-2
Browse files Browse the repository at this point in the history
Add podman farm build command
  • Loading branch information
openshift-ci[bot] authored Oct 25, 2023
2 parents 32ef2d7 + ebe01ca commit 5a47b1e
Show file tree
Hide file tree
Showing 37 changed files with 1,969 additions and 31 deletions.
19 changes: 19 additions & 0 deletions .cirrus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -910,6 +910,24 @@ minikube_test_task:
main_script: *main
always: *logs_artifacts

farm_test_task:
name: *std_name_fmt
alias: farm_test
# Docs: ./contrib/cirrus/CIModes.md
only_if: *not_tag_build_docs
depends_on:
- build
- rootless_system_test
gce_instance: *standardvm
env:
<<: *stdenvars
TEST_FLAVOR: farm
PRIV_NAME: rootless
clone_script: *get_gosrc
setup_script: *setup
main_script: *main
always: *logs_artifacts

buildah_bud_test_task:
name: *std_name_fmt
alias: buildah_bud_test
Expand Down Expand Up @@ -1054,6 +1072,7 @@ success_task:
- rootless_system_test
- rootless_remote_system_test
- minikube_test
- farm_test
- buildah_bud_test
- rootless_buildah_bud_test
- upgrade_test
Expand Down
13 changes: 12 additions & 1 deletion cmd/podman/common/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,13 @@ type BuildFlagsWrapper struct {
Cleanup bool
}

func DefineBuildFlags(cmd *cobra.Command, buildOpts *BuildFlagsWrapper) {
// FarmBuildHiddenFlags are the flags hidden from the farm build command because they are either not
// supported or don't make sense in the farm build use case
var FarmBuildHiddenFlags = []string{"arch", "all-platforms", "compress", "cw", "disable-content-trust",
"logsplit", "manifest", "os", "output", "platform", "sign-by", "signature-policy", "stdin", "tls-verify",
"variant"}

func DefineBuildFlags(cmd *cobra.Command, buildOpts *BuildFlagsWrapper, isFarmBuild bool) {
flags := cmd.Flags()

// buildx build --load ignored, but added for compliance
Expand Down Expand Up @@ -116,6 +122,11 @@ func DefineBuildFlags(cmd *cobra.Command, buildOpts *BuildFlagsWrapper) {
_ = flags.MarkHidden("logsplit")
_ = flags.MarkHidden("cw")
}
if isFarmBuild {
for _, f := range FarmBuildHiddenFlags {
_ = flags.MarkHidden(f)
}
}
}

func ParseBuildOpts(cmd *cobra.Command, args []string, buildOpts *BuildFlagsWrapper) (*entities.BuildOptions, error) {
Expand Down
135 changes: 135 additions & 0 deletions cmd/podman/farm/build.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
package farm

import (
"errors"
"fmt"
"os"

"github.com/containers/common/pkg/config"
"github.com/containers/podman/v4/cmd/podman/common"
"github.com/containers/podman/v4/cmd/podman/registry"
"github.com/containers/podman/v4/cmd/podman/utils"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/containers/podman/v4/pkg/farm"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)

type buildOptions struct {
buildOptions common.BuildFlagsWrapper
local bool
platforms []string
}

var (
farmBuildDescription = `Build images on farm nodes, then bundle them into a manifest list`
buildCommand = &cobra.Command{
Use: "build [options] [CONTEXT]",
Short: "Build a container image for multiple architectures",
Long: farmBuildDescription,
RunE: build,
Example: "podman farm build [flags] buildContextDirectory",
Args: cobra.ExactArgs(1),
}
buildOpts = buildOptions{
buildOptions: common.BuildFlagsWrapper{},
}
)

func init() {
registry.Commands = append(registry.Commands, registry.CliCommand{
Command: buildCommand,
Parent: farmCmd,
})
flags := buildCommand.Flags()
flags.SetNormalizeFunc(utils.AliasFlags)

localFlagName := "local"
// Default for local is true and hide this flag for the remote use case
if !registry.IsRemote() {
flags.BoolVarP(&buildOpts.local, localFlagName, "l", true, "Build image on local machine as well as on farm nodes")
}
cleanupFlag := "cleanup"
flags.BoolVar(&buildOpts.buildOptions.Cleanup, cleanupFlag, false, "Remove built images from farm nodes on success")
platformsFlag := "platforms"
buildCommand.PersistentFlags().StringSliceVar(&buildOpts.platforms, platformsFlag, nil, "Build only on farm nodes that match the given platforms")

common.DefineBuildFlags(buildCommand, &buildOpts.buildOptions, true)
}

func build(cmd *cobra.Command, args []string) error {
// Return error if any of the hidden flags are used
for _, f := range common.FarmBuildHiddenFlags {
if cmd.Flags().Changed(f) {
return fmt.Errorf("%q is an unsupported flag for podman farm build", f)
}
}

if !cmd.Flags().Changed("tag") {
return errors.New("cannot create manifest list without a name, value for --tag is required")
}
opts, err := common.ParseBuildOpts(cmd, args, &buildOpts.buildOptions)
if err != nil {
return err
}
// Close the logFile if one was created based on the flag
if opts.LogFileToClose != nil {
defer opts.LogFileToClose.Close()
}
if opts.TmpDirToClose != "" {
// We had to download the context directory.
// Delete it later.
defer func() {
if err = os.RemoveAll(opts.TmpDirToClose); err != nil {
logrus.Errorf("Removing temporary directory %q: %v", opts.TmpDirToClose, err)
}
}()
}
opts.Cleanup = buildOpts.buildOptions.Cleanup
iidFile, err := cmd.Flags().GetString("iidfile")
if err != nil {
return err
}
opts.IIDFile = iidFile

cfg, err := config.ReadCustomConfig()
if err != nil {
return err
}

defaultFarm := cfg.Farms.Default
if farmCmd.Flags().Changed("farm") {
f, err := farmCmd.Flags().GetString("farm")
if err != nil {
return err
}
defaultFarm = f
}

var localEngine entities.ImageEngine
if buildOpts.local {
localEngine = registry.ImageEngine()
}

ctx := registry.Context()
farm, err := farm.NewFarm(ctx, defaultFarm, localEngine)
if err != nil {
return fmt.Errorf("initializing: %w", err)
}

schedule, err := farm.Schedule(ctx, buildOpts.platforms)
if err != nil {
return fmt.Errorf("scheduling builds: %w", err)
}
logrus.Infof("schedule: %v", schedule)

manifestName := opts.Output
// Set Output to "" so that the images built on the farm nodes have no name
opts.Output = ""
if err = farm.Build(ctx, schedule, *opts, manifestName); err != nil {
return fmt.Errorf("build: %w", err)
}
logrus.Infof("build: ok")

return nil
}
9 changes: 1 addition & 8 deletions cmd/podman/farm/farm.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ var (
var (
// Temporary struct to hold cli values.
farmOpts = struct {
Farm string
Local bool
Farm string
}{}
)

Expand All @@ -40,10 +39,4 @@ func init() {
defaultFarm = podmanConfig.ContainersConfDefaultsRO.Farms.Default
}
flags.StringVarP(&farmOpts.Farm, farmFlagName, "f", defaultFarm, "Farm to use for builds")

localFlagName := "local"
// Default for local is true and hide this flag for the remote use case
if !registry.IsRemote() {
flags.BoolVarP(&farmOpts.Local, localFlagName, "l", true, "Build image on local machine including on farm nodes")
}
}
2 changes: 1 addition & 1 deletion cmd/podman/images/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func init() {
}

func buildFlags(cmd *cobra.Command) {
common.DefineBuildFlags(cmd, &buildOpts)
common.DefineBuildFlags(cmd, &buildOpts, false)
}

// build executes the build command.
Expand Down
13 changes: 12 additions & 1 deletion contrib/cirrus/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,11 @@ setup_rootless() {
# shellcheck disable=SC2154
if passwd --status $ROOTLESS_USER
then
if [[ $PRIV_NAME = "rootless" ]]; then
# Farm tests utilize the rootless user to simulate a "remote" podman instance.
# Root still needs to own the repo. clone and all things under `$GOPATH`. The
# opposite is true for the lower-level podman e2e tests, the rootless user
# runs them, and therefore needs permissions.
if [[ $PRIV_NAME = "rootless" ]] && [[ "$TEST_FLAVOR" != "farm" ]]; then
msg "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC"
return 0
Expand Down Expand Up @@ -184,6 +188,13 @@ setup_rootless() {
# Maintain access-permission consistency with all other .ssh files.
install -Z -m 700 -o $ROOTLESS_USER -g $ROOTLESS_USER \
/root/.ssh/known_hosts /home/$ROOTLESS_USER/.ssh/known_hosts

if [[ -n "$ROOTLESS_USER" ]]; then
showrun echo "conditional setup for ROOTLESS_USER [=$ROOTLESS_USER]"
# Make all future CI scripts aware of these values
echo "ROOTLESS_USER=$ROOTLESS_USER" >> /etc/ci_environment
echo "ROOTLESS_UID=$ROOTLESS_UID" >> /etc/ci_environment
fi
}

install_test_configs() {
Expand Down
6 changes: 6 additions & 0 deletions contrib/cirrus/runner.sh
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ function _run_minikube() {
showrun bats test/minikube |& logformatter
}

function _run_farm() {
_bail_if_test_can_be_skipped test/farm test/system
msg "Testing podman farm."
showrun bats test/farm |& logformatter
}

exec_container() {
local var_val
local cmd
Expand Down
14 changes: 7 additions & 7 deletions contrib/cirrus/setup_environment.sh
Original file line number Diff line number Diff line change
Expand Up @@ -269,13 +269,6 @@ case "$PRIV_NAME" in
*) die_unknown PRIV_NAME
esac

# shellcheck disable=SC2154
if [[ -n "$ROOTLESS_USER" ]]; then
showrun echo "conditional setup for ROOTLESS_USER [=$ROOTLESS_USER]"
echo "ROOTLESS_USER=$ROOTLESS_USER" >> /etc/ci_environment
echo "ROOTLESS_UID=$ROOTLESS_UID" >> /etc/ci_environment
fi

# FIXME! experimental workaround for #16973, the "lookup cdn03.quay.io" flake.
#
# If you are reading this on or after April 2023:
Expand Down Expand Up @@ -403,6 +396,13 @@ case "$TEST_FLAVOR" in
die "Invalid value for \$TEST_ENVIRON=$TEST_ENVIRON"
fi

install_test_configs
;;
farm)
showrun loginctl enable-linger $ROOTLESS_USER
showrun ssh $ROOTLESS_USER@localhost systemctl --user enable --now podman.socket
remove_packaged_podman_files
showrun make install PREFIX=/usr ETCDIR=/etc
install_test_configs
;;
minikube)
Expand Down
1 change: 1 addition & 0 deletions libpod/define/info.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ type HostInfo struct {
SwapFree int64 `json:"swapFree"`
SwapTotal int64 `json:"swapTotal"`
Uptime string `json:"uptime"`
Variant string `json:"variant"`
Linkmode string `json:"linkmode"`
}

Expand Down
6 changes: 6 additions & 0 deletions libpod/info.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"time"

"github.com/containers/buildah"
"github.com/containers/buildah/pkg/parse"
"github.com/containers/buildah/pkg/util"
"github.com/containers/common/pkg/version"
"github.com/containers/image/v5/pkg/sysregistriesv2"
Expand Down Expand Up @@ -130,6 +131,11 @@ func (r *Runtime) hostInfo() (*define.HostInfo, error) {
SwapFree: mi.SwapFree,
SwapTotal: mi.SwapTotal,
}
platform := parse.DefaultPlatform()
pArr := strings.Split(platform, "/")
if len(pArr) == 3 {
info.Variant = pArr[2]
}
if err := r.setPlatformHostInfo(&info); err != nil {
return nil, err
}
Expand Down
16 changes: 11 additions & 5 deletions pkg/bindings/images/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (

"github.com/containers/buildah/define"
"github.com/containers/image/v5/types"
ldefine "github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/auth"
"github.com/containers/podman/v4/pkg/bindings"
"github.com/containers/podman/v4/pkg/domain/entities"
Expand Down Expand Up @@ -500,6 +501,11 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
}
}

saveFormat := ldefine.OCIArchive
if options.OutputFormat == define.Dockerv2ImageManifest {
saveFormat = ldefine.V2s2Archive
}

// build secrets are usually absolute host path or relative to context dir on host
// in any case move secret to current context and ship the tar.
if secrets := options.CommonBuildOpts.Secrets; len(secrets) > 0 {
Expand Down Expand Up @@ -602,7 +608,7 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
// even when the server quit but it seems desirable to
// distinguish a proper build from a transient EOF.
case <-response.Request.Context().Done():
return &entities.BuildReport{ID: id}, nil
return &entities.BuildReport{ID: id, SaveFormat: saveFormat}, nil
default:
// non-blocking select
}
Expand All @@ -616,7 +622,7 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
if errors.Is(err, io.EOF) && id != "" {
break
}
return &entities.BuildReport{ID: id}, fmt.Errorf("decoding stream: %w", err)
return &entities.BuildReport{ID: id, SaveFormat: saveFormat}, fmt.Errorf("decoding stream: %w", err)
}

switch {
Expand All @@ -629,12 +635,12 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
case s.Error != "":
// If there's an error, return directly. The stream
// will be closed on return.
return &entities.BuildReport{ID: id}, errors.New(s.Error)
return &entities.BuildReport{ID: id, SaveFormat: saveFormat}, errors.New(s.Error)
default:
return &entities.BuildReport{ID: id}, errors.New("failed to parse build results stream, unexpected input")
return &entities.BuildReport{ID: id, SaveFormat: saveFormat}, errors.New("failed to parse build results stream, unexpected input")
}
}
return &entities.BuildReport{ID: id}, nil
return &entities.BuildReport{ID: id, SaveFormat: saveFormat}, nil
}

func nTar(excludes []string, sources ...string) (io.ReadCloser, error) {
Expand Down
4 changes: 3 additions & 1 deletion pkg/bindings/test/images_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"time"

podmanRegistry "github.com/containers/podman/v4/hack/podman-registry-go"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/bindings"
"github.com/containers/podman/v4/pkg/bindings/containers"
"github.com/containers/podman/v4/pkg/bindings/images"
Expand Down Expand Up @@ -409,7 +410,8 @@ var _ = Describe("Podman images", func() {
results, err := images.Build(bt.conn, []string{"fixture/Containerfile"}, entities.BuildOptions{})
Expect(err).ToNot(HaveOccurred())
Expect(*results).To(MatchFields(IgnoreMissing, Fields{
"ID": Not(BeEmpty()),
"ID": Not(BeEmpty()),
"SaveFormat": ContainSubstring(define.OCIArchive),
}))
})
})
Loading

1 comment on commit 5a47b1e

@packit-as-a-service
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

podman-next COPR build failed. @containers/packit-build please check.

Please sign in to comment.