AWS boot testing for all supported image types

HACKING.md

@@ -82,10 +82,88 @@ cache and if these changes are made as root, it can cause issues when running
 other go commands in the future as a regular user. Instead, it is recommended
 to first build the binary and then run it as root:
 ```
-go build -o build ./cmd/build
-sudo ./build ...
+go build -o bin/build ./cmd/build
+sudo ./bin/build ...
 ```
 
+#### Booting images
+
+You can boot an image in its target environment by using the appropriate
+command from `cmd/`. _Currently, only AWS is supported._
+
+For example, to boot an AMI or EC2 image, you can use the `./cmd/boot-aws`
+command with the `setup` subcommand:
+```bash
+go run ./cmd/boot-aws setup \
+     --access-key-id "${AWS_ACCESS_KEY_ID}" \
+     --secret-access-key "${AWS_SECRET_ACCESS_KEY}" \
+     --region "${AWS_REGION}" \
+     --bucket "${AWS_BUCKET}" \
+     --ami-name "${IMAGE_NAME}" \
+     --s3-key "${IMAGE_KEY}" \
+     --username "${USERNAME}" \
+     --arch "${IMAGE_ARCHITECTURE}" \
+     --ssh-pubkey "${PATH_TO_SSH_PUBLIC_KEY}" \
+     --ssh-privkey "${PATH_TO_SSH_PRIVATE_KEY}" \
+     --resourcefile ./aws-test-resources.json \
+     ${PATH_TO_IMAGE_FILE}
+```
+where:
+- `${AWS_ACCESS_KEY_ID}` and `${AWS_SECRET_ACCESS_KEY}` are the AWS credentials,
+- `${AWS_REGION}` is the AWS region to use,
+- `${AWS_BUCKET}` is an S3 bucket (that must already exist),
+- `${IMAGE_NAME}` is the name to use for registering the AMI,
+- `${IMAGE_KEY}` is the key (filename) to use for the file in S3,
+- `${USERNAME}` is the username to set up on the instance,
+- `${IMAGE_ARCHITECTURE}` is the hardware architecture of the image being
+  uploaded and booted,
+- `${PATH_TO_SSH_PUBLIC_KEY}` and `${PATH_TO_SSH_PRIVATE_KEY}` point to an
+  public/private SSH key pair.
+
+This command will upload the image to S3, register the image as an AMI, create
+a security group configured to allow SSH access, and launch an instance from
+the AMI. It will then wait until the instance is ready and print its public IP
+address. It will also use the public ssh key and provided username to configure
+cloud-init to create a user and set the ssh key on first boot.
+
+The IDs of all created resources are stored in the file specified by the
+`--resourcefile` flag. This can be used to tear down all the resources created
+by the `setup` subcommand:
+```bash
+go run ./cmd/boot-aws teardown \
+     --access-key-id "${AWS_ACCESS_KEY_ID}" \
+     --secret-access-key "${AWS_SECRET_ACCESS_KEY}" \
+     --region "${AWS_REGION}" \
+     --bucket "${AWS_BUCKET}" \
+     --name "${IMAGE_NAME}" \
+     --key "${IMAGE_KEY}" \
+     --username "${USERNAME}" \
+     --arch "${IMAGE_ARCHITECTURE}" \
+     --ssh-pubkey "${PATH_TO_SSH_PUBLIC_KEY}" \
+     --ssh-privkey "${PATH_TO_SSH_PRIVATE_KEY}" \
+     --resourcefile ./aws-test-resources.json
+```
+
+Alternatively, a setup-test-teardown procedure can be run in a single command using the `run` subcommand:
+```bash
+go run ./cmd/boot-aws run \
+     --access-key-id "${AWS_ACCESS_KEY_ID}" \
+     --secret-access-key "${AWS_SECRET_ACCESS_KEY}" \
+     --region "${AWS_REGION}" \
+     --bucket "${AWS_BUCKET}" \
+     --ami-name "${IMAGE_NAME}" \
+     --s3-key "${IMAGE_KEY}" \
+     --username "${USERNAME}" \
+     --arch "${IMAGE_ARCHITECTURE}" \
+     --ssh-pubkey "${PATH_TO_SSH_PUBLIC_KEY}" \
+     --ssh-privkey "${PATH_TO_SSH_PRIVATE_KEY}" \
+     ${PATH_TO_IMAGE_FILE} ${PATH_TO_SCRIPT}
+```
+
+This will perform the same steps as the `setup` subcommand, then upload the
+script specified by `${PATH_TO_SCRIPT}` to the instance, run it, and then
+perform the same actions as the `teardown` subcommand.
+
 #### Listing available image type configurations
 
 The `cmd/list-images` utility simply lists all available combinations of

cmd/boot-aws/main.go

@@ -107,7 +107,7 @@ func scpFile(ip, user, key, hostsfile, source, dest string) error {
 
 func keyscan(ip, filepath string) error {
 	var keys []byte
-	maxTries := 10
+	maxTries := 30 // wait for at least 5 mins
 	var keyscanErr error
 	for try := 0; try < maxTries; try++ {
 		keys, _, keyscanErr = run("ssh-keyscan", ip)
@@ -159,12 +159,12 @@ func doSetup(a *awscloud.AWS, filename string, flags *pflag.FlagSet, res *resour
 	if err != nil {
 		return err
 	}
-	sshKey, err := flags.GetString("ssh-key")
+	sshPubKey, err := flags.GetString("ssh-pubkey")
 	if err != nil {
 		return err
 	}
 
-	userData, err := createUserData(username, sshKey)
+	userData, err := createUserData(username, sshPubKey)
 	if err != nil {
 		return fmt.Errorf("createUserData(): %s", err.Error())
 	}
@@ -173,7 +173,7 @@ func doSetup(a *awscloud.AWS, filename string, flags *pflag.FlagSet, res *resour
 	if err != nil {
 		return err
 	}
-	keyName, err := flags.GetString("key")
+	keyName, err := flags.GetString("s3-key")
 	if err != nil {
 		return err
 	}
@@ -185,21 +185,14 @@ func doSetup(a *awscloud.AWS, filename string, flags *pflag.FlagSet, res *resour
 
 	fmt.Printf("file uploaded to %s\n", aws.StringValue(&uploadOutput.Location))
 
-	var share []string
-	if shareWith, err := flags.GetString("account-id"); shareWith != "" {
-		share = append(share, shareWith)
-	} else if err != nil {
-		return err
-	}
-
 	var bootModePtr *string
 	if bootMode, err := flags.GetString("boot-mode"); bootMode != "" {
 		bootModePtr = &bootMode
 	} else if err != nil {
 		return err
 	}
 
-	imageName, err := flags.GetString("name")
+	imageName, err := flags.GetString("ami-name")
 	if err != nil {
 		return err
 	}
@@ -209,7 +202,7 @@ func doSetup(a *awscloud.AWS, filename string, flags *pflag.FlagSet, res *resour
 		return err
 	}
 
-	ami, snapshot, err := a.Register(imageName, bucketName, keyName, share, arch, bootModePtr)
+	ami, snapshot, err := a.Register(imageName, bucketName, keyName, nil, arch, bootModePtr)
 	if err != nil {
 		return fmt.Errorf("Register(): %s", err.Error())
 	}
@@ -366,7 +359,7 @@ func teardown(cmd *cobra.Command, args []string) {
 }
 
 func doRunExec(a *awscloud.AWS, filename string, flags *pflag.FlagSet, res *resources) error {
-	sshKey, err := flags.GetString("ssh-key")
+	privKey, err := flags.GetString("ssh-privkey")
 	if err != nil {
 		return err
 	}
@@ -391,24 +384,21 @@ func doRunExec(a *awscloud.AWS, filename string, flags *pflag.FlagSet, res *reso
 		return err
 	}
 
-	// TODO: remove this assumption and add a private key flag
-	key := strings.TrimSuffix(sshKey, ".pub")
-
 	// ssh into the remote machine and exit immediately to check connection
-	if err := sshRun(ip, username, key, hostsfile, "exit"); err != nil {
+	if err := sshRun(ip, username, privKey, hostsfile, "exit"); err != nil {
 		return err
 	}
 
 	// copy the executable without its path to the remote host
 	destination := filepath.Base(filename)
 
 	// copy the executable
-	if err := scpFile(ip, username, key, hostsfile, filename, destination); err != nil {
+	if err := scpFile(ip, username, privKey, hostsfile, filename, destination); err != nil {
 		return err
 	}
 
 	// run the executable
-	return sshRun(ip, username, key, hostsfile, fmt.Sprintf("./%s", destination))
+	return sshRun(ip, username, privKey, hostsfile, fmt.Sprintf("./%s", destination))
 }
 
 func runExec(cmd *cobra.Command, args []string) {
@@ -454,33 +444,34 @@ func setupCLI() *cobra.Command {
 	rootFlags.String("session-token", "", "session token")
 	rootFlags.String("region", "", "target region")
 	rootFlags.String("bucket", "", "target S3 bucket name")
-	rootFlags.String("key", "", "target S3 key name")
-	rootFlags.String("name", "", "AMI name")
-	rootFlags.String("account-id", "", "account id to share image with")
+	rootFlags.String("s3-key", "", "target S3 key name")
+	rootFlags.String("ami-name", "", "AMI name")
 	rootFlags.String("arch", "", "arch (x86_64 or aarch64)")
 	rootFlags.String("boot-mode", "", "boot mode (legacy-bios, uefi, uefi-preferred)")
 	rootFlags.String("username", "", "name of the user to create on the system")
-	rootFlags.String("ssh-key", "", "path to user's public ssh key")
+	rootFlags.String("ssh-pubkey", "", "path to user's public ssh key")
+	rootFlags.String("ssh-privkey", "", "path to user's private ssh key")
 
 	exitCheck(rootCmd.MarkPersistentFlagRequired("access-key-id"))
 	exitCheck(rootCmd.MarkPersistentFlagRequired("secret-access-key"))
 	exitCheck(rootCmd.MarkPersistentFlagRequired("region"))
 	exitCheck(rootCmd.MarkPersistentFlagRequired("bucket"))
 
 	// TODO: make it optional and use UUID if not specified
-	exitCheck(rootCmd.MarkPersistentFlagRequired("key"))
+	exitCheck(rootCmd.MarkPersistentFlagRequired("s3-key"))
 
 	// TODO: make it optional and use UUID if not specified
-	exitCheck(rootCmd.MarkPersistentFlagRequired("name"))
+	exitCheck(rootCmd.MarkPersistentFlagRequired("ami-name"))
 
 	exitCheck(rootCmd.MarkPersistentFlagRequired("arch"))
 
 	// TODO: make it optional and use a default
 	exitCheck(rootCmd.MarkPersistentFlagRequired("username"))
 
-	// TODO: make ssh key optional for 'run' and if not specified generate a
-	// temporary key pair
-	exitCheck(rootCmd.MarkPersistentFlagRequired("ssh-key"))
+	// TODO: make ssh key pair optional for 'run' and if not specified generate
+	// a temporary key pair
+	exitCheck(rootCmd.MarkPersistentFlagRequired("ssh-privkey"))
+	exitCheck(rootCmd.MarkPersistentFlagRequired("ssh-pubkey"))
 
 	setupCmd := &cobra.Command{
 		Use:                   "setup [--resourcefile <filename>] <filename>",

pkg/distro/rhel8/edge.go

@@ -65,12 +65,13 @@ func edgeRawImgType() imageType {
 		filename:            "image.raw.xz",
 		compression:         "xz",
 		mimeType:            "application/xz",
+		image:               edgeRawImage,
 		packageSets:         nil,
 		defaultSize:         10 * common.GibiByte,
 		rpmOstree:           true,
 		bootable:            true,
 		bootISO:             false,
-		image:               edgeRawImage,
+		kernelOptions:       "modprobe.blacklist=vc4",
 		buildPipelines:      []string{"build"},
 		payloadPipelines:    []string{"ostree-deployment", "image", "xz"},
 		exports:             []string{"xz"},

pkg/distro/rhel9/ami.go

@@ -8,6 +8,7 @@ import (
 	"github.com/osbuild/images/pkg/subscription"
 )
 
+// TODO: move these to the EC2 environment
 const amiKernelOptions = "console=ttyS0,115200n8 console=tty0 net.ifnames=0 rd.blacklist=nouveau nvme_core.io_timeout=4294967295"
 
 var (

pkg/distro/rhel9/edge.go

@@ -69,6 +69,7 @@ var (
 		defaultImageConfig: &distro.ImageConfig{
 			Locale: common.ToPtr("en_US.UTF-8"),
 		},
+		kernelOptions:       "modprobe.blacklist=vc4",
 		defaultSize:         10 * common.GibiByte,
 		rpmOstree:           true,
 		bootable:            true,
@@ -145,6 +146,7 @@ var (
 		defaultImageConfig: &distro.ImageConfig{
 			Locale: common.ToPtr("en_US.UTF-8"),
 		},
+		kernelOptions:       amiKernelOptions + " modprobe.blacklist=vc4",
 		defaultSize:         10 * common.GibiByte,
 		rpmOstree:           true,
 		bootable:            true,
@@ -165,6 +167,7 @@ var (
 		defaultImageConfig: &distro.ImageConfig{
 			Locale: common.ToPtr("en_US.UTF-8"),
 		},
+		kernelOptions:       "modprobe.blacklist=vc4",
 		defaultSize:         10 * common.GibiByte,
 		rpmOstree:           true,
 		bootable:            true,

pkg/distro/rhel9/images.go

@@ -405,7 +405,12 @@ func edgeRawImage(workload workload.Workload,
 	img.Users = users.UsersFromBP(customizations.GetUsers())
 	img.Groups = users.GroupsFromBP(customizations.GetGroups())
 
-	img.KernelOptionsAppend = []string{"modprobe.blacklist=vc4"}
+	// The kernel options defined on the image type are usually handled in
+	// osCustomiztions() but ostree images don't use OSCustomizations, so we
+	// handle them here separately.
+	if t.kernelOptions != "" {
+		img.KernelOptionsAppend = append(img.KernelOptionsAppend, t.kernelOptions)
+	}
 	img.Keyboard = "us"
 	img.Locale = "C.UTF-8"
 	if !common.VersionLessThan(t.arch.distro.osVersion, "9.2") || !t.arch.distro.isRHEL() {

test/README.md

@@ -4,7 +4,8 @@
 
 - `./cmd/build` takes a config file as argument to build an image.  For example:
 ```
-sudo go run ./cmd/build -output ./buildtest -rpmmd /tmp/rpmmd -distro fedora-38 -image qcow2 -config test/configs/embed-containers.json
+go build -o bin/build ./cmd/build
+sudo ./bin/build -output ./buildtest -rpmmd /tmp/rpmmd -distro fedora-38 -image qcow2 -config test/configs/embed-containers.json
 ```
 will build a Fedora 38 qcow2 image using the configuration specified in the file `embed-containers.json`
 
@@ -29,6 +30,7 @@ configure-generators
     |         | (Dynamic: For each modified image type and config)
     |         |-- Build <distro>-<arch>-<image>-<config>
     |
+    | (Dynamic: For each distro/arch)
     |-- generate-ostree-build-configs-<distro>-<arch>
               |
               | (Dynamic: For each modified image type and config)
@@ -54,16 +56,18 @@ The config generator:
 - Downloads the test build cache.
 - Filters out any manifest with an ID that exists in the build cache.
   - It also filters out any manifest that depends on an ostree commit because these can't be built without an ostree repository to pull from.
-- For each remaining manifest, creates a build job which runs the `./test/scripts/build-image.sh` script for a given distro, image type, and config file.
+- For each remaining manifest, creates a job which builds, boots (if applicable), and uploads the results to the build cache for a given distro, image type, and config file.
+  - `./test/scripts/build-image` builds the image using osbuild.
+  - `./test/scripts/boot-image` boots the image in the appropriate cloud or virtual environment (if supported).
+  - `./test/scripts/upload-results` uploads the results (manifest, image file, and build info) to the CI S3 bucket, so that rebuilds of the same manifest ID can be skipped.
   - For ostree container image types (`iot-container` and `edge-container`), it also adds a call to the `./tools/ci/push-container.sh` script to push the container to the GitLab registry. The name and tag for each container is `<build name>:<manifest ID>` (see [Definitions](#definitions) below).
-  - If no builds are needed, it generates a `NullConfig`, which is a simple shell runner that exits successfully. This is required because the child pipeline config cannot be empty.
-
+- If no builds are needed, it generates a `NullConfig`, which is a simple shell runner that exits successfully. This is required because the child pipeline config cannot be empty.
 
 #### 2. Dynamic build job
 
 Each build job runs in parallel. For each image that is successfully built, a file is added to the test build cache under the following path:
 ```
-<distro>/<arch>/<manifest ID>.json
+<distro>/<arch>/<manifest ID>/info.json
 ```
 
 Each file in the cache stores information relevant to the build,
@@ -76,7 +80,8 @@ in the form
   "config": "<config name>",
   "manifest-checksum": "<manifest ID>",
   "obuild-version": "<osbuild version>",
-  "commit": "<commit ID>"
+  "commit": "<commit ID>",
+  "pr": "<PR number>"
 }
 ```
 
@@ -91,7 +96,8 @@ for example:
   "config": "all-customizations",
   "manifest-checksum": "8c0ce3987d78fe6f3307494cd57ceed861de61c3b04786d6a7f570faacbdb5df",
   "obuild-version": "osbuild 89",
-  "commit": "52ecfdf1eb345e09c6a6edf4a8d3dd5c8079c51c"
+  "commit": "52ecfdf1eb345e09c6a6edf4a8d3dd5c8079c51c",
+  "pr": 42
 }
 ```
 
@@ -111,8 +117,11 @@ The config generator:
   - Note that this manifest generation step uses the `-skip-noconfig` flag, which means that any image type not defined in the map is skipped.
 - Downloads the test build cache.
 - Filters out any manifest with an ID that exists in the build cache.
-- For each remaining manifest, creates a build job which runs the ostree container that was used to generate the manifest and runs `./test/scripts/build-image.sh` script for a given distro, image type, and config file.
-  - If no builds are needed, it generates a `NullConfig`, which is a simple shell runner that exits successfully. This is required because the child pipeline config cannot be empty.
+- For each remaining manifest, creates a job which builds, boots (if applicable), and uploads the results to the build cache for a given distro, image type, and config file.
+  - `./test/scripts/build-image` builds the image using osbuild.
+  - `./test/scripts/boot-image` boots the image in the appropriate cloud or virtual environment (if supported).
+  - `./test/scripts/upload-results` uploads the results (manifest, image file, and build info) to the CI S3 bucket, so that rebuilds of the same manifest ID can be skipped.
+- If no builds are needed, it generates a `NullConfig`, which is a simple shell runner that exits successfully. This is required because the child pipeline config cannot be empty.
 
 
 #### 4. Dynamic ostree build job