Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for building images from ostree containers #272

Merged
merged 17 commits into from
Nov 28, 2023
Merged

Add support for building images from ostree containers #272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
0464233
osbuild: add Len() to the ContainersInputReferences interface
achilleas-k Oct 30, 2023
d7058d3
osbuild: add ostree.deploy.container stage
achilleas-k Oct 30, 2023
0bab73b
manifest: only set one ostree commit in OSTreeDeployment
achilleas-k Nov 3, 2023
e21cec1
manifest: add a container source to OSTreeDeployment
achilleas-k Nov 3, 2023
cdcc96b
manifest: run ostree pull right before ostree deploy
achilleas-k Nov 6, 2023
a4bd65d
manifest: add ostree deploy stages conditionally
achilleas-k Nov 6, 2023
9717175
manifest: add a second constructor for ostree container deployment
achilleas-k Nov 6, 2023
c8ea2b0
manifest: add ostree.deploy.container stage to ostree deployment
achilleas-k Nov 6, 2023
a85b7ae
image: support both commit and container sources for OSTreeDiskImage
achilleas-k Nov 6, 2023
404475c
manifest: remove ignition options from OSTreeDeployment constructor
achilleas-k Nov 6, 2023
b43ff16
cmd: use internal types for config and repositories
achilleas-k Nov 6, 2023
bebad6c
test: update repository format
achilleas-k Nov 6, 2023
a7d1c6e
manifest: move root user locking into image function
achilleas-k Nov 6, 2023
51bf424
manifest: move coreos.no_persist_ip into image functions
achilleas-k Nov 24, 2023
4cca349
manifest: set rootfs label in ostree deployment
achilleas-k Nov 10, 2023
b497822
manifest: refactor ostree deployment stages into functions
achilleas-k Nov 27, 2023
a91eafe
test: update terraform sha
achilleas-k Nov 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 10 additions & 90 deletions cmd/build/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,47 +34,11 @@ func check(err error) {
}
}

type repository struct {
achilleas-k marked this conversation as resolved.
Show resolved Hide resolved
Name string `json:"name"`
Id string `json:"id,omitempty"`
BaseURL string `json:"baseurl,omitempty"`
Metalink string `json:"metalink,omitempty"`
MirrorList string `json:"mirrorlist,omitempty"`
GPGKey string `json:"gpgkey,omitempty"`
CheckGPG bool `json:"check_gpg,omitempty"`
CheckRepoGPG bool `json:"check_repo_gpg,omitempty"`
IgnoreSSL bool `json:"ignore_ssl,omitempty"`
RHSM bool `json:"rhsm,omitempty"`
MetadataExpire string `json:"metadata_expire,omitempty"`
ImageTypeTags []string `json:"image_type_tags,omitempty"`
PackageSets []string `json:"package-sets,omitempty"`
}

type ostreeOptions struct {
Ref string `json:"ref"`
URL string `json:"url"`
Parent string `json:"parent"`
RHSM bool `json:"rhsm"`
}

type crBlueprint struct {
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
Version string `json:"version,omitempty"`
Packages []blueprint.Package `json:"packages,omitempty"`
Modules []blueprint.Package `json:"modules,omitempty"`
Groups []blueprint.Group `json:"groups,omitempty"`
Containers []blueprint.Container `json:"containers,omitempty"`
Customizations *blueprint.Customizations `json:"customizations,omitempty"`
Distro string `json:"distro,omitempty"`
Minimal bool `json:"minimal,omitempty"`
}

type BuildConfig struct {
Name string `json:"name"`
OSTree *ostreeOptions `json:"ostree,omitempty"`
Blueprint *crBlueprint `json:"blueprint,omitempty"`
Depends interface{} `json:"depends,omitempty"` // ignored
Name string `json:"name"`
OSTree *ostree.ImageOptions `json:"ostree,omitempty"`
Blueprint *blueprint.Blueprint `json:"blueprint,omitempty"`
Depends interface{} `json:"depends,omitempty"` // ignored
}

func loadConfig(path string) BuildConfig {
Expand All @@ -97,14 +61,7 @@ func makeManifest(imgType distro.ImageType, config BuildConfig, distribution dis
cacheDir := filepath.Join(cacheRoot, archName+distribution.Name())

options := distro.ImageOptions{Size: 0}
if config.OSTree != nil {
options.OSTree = &ostree.ImageOptions{
URL: config.OSTree.URL,
ImageRef: config.OSTree.Ref,
ParentRef: config.OSTree.Parent,
RHSM: config.OSTree.RHSM,
}
}
options.OSTree = config.OSTree

// add RHSM fact to detect changes
options.Facts = &facts.ImageOptions{
Expand Down Expand Up @@ -154,43 +111,7 @@ func makeManifest(imgType distro.ImageType, config BuildConfig, distribution dis
return mf, nil
}

type DistroArchRepoMap map[string]map[string][]repository

func convertRepo(r repository) rpmmd.RepoConfig {
var urls []string
if r.BaseURL != "" {
urls = []string{r.BaseURL}
}

var keys []string
if r.GPGKey != "" {
keys = []string{r.GPGKey}
}

return rpmmd.RepoConfig{
Id: r.Id,
Name: r.Name,
BaseURLs: urls,
Metalink: r.Metalink,
MirrorList: r.MirrorList,
GPGKeys: keys,
CheckGPG: &r.CheckGPG,
CheckRepoGPG: &r.CheckRepoGPG,
IgnoreSSL: &r.IgnoreSSL,
MetadataExpire: r.MetadataExpire,
RHSM: r.RHSM,
ImageTypeTags: r.ImageTypeTags,
PackageSets: r.PackageSets,
}
}

func convertRepos(rr []repository) []rpmmd.RepoConfig {
cr := make([]rpmmd.RepoConfig, len(rr))
for idx, r := range rr {
cr[idx] = convertRepo(r)
}
return cr
}
type DistroArchRepoMap map[string]map[string][]rpmmd.RepoConfig

func readRepos() DistroArchRepoMap {
reposDir := "./test/data/repositories/"
Expand All @@ -208,7 +129,7 @@ func readRepos() DistroArchRepoMap {
defer fp.Close()
data, err := io.ReadAll(fp)
check(err)
repos := make(map[string][]repository)
repos := make(map[string][]rpmmd.RepoConfig)
check(json.Unmarshal(data, &repos))
distro := strings.TrimSuffix(filename, filepath.Ext(filename))
darm[distro] = repos
Expand Down Expand Up @@ -289,8 +210,8 @@ func u(s string) string {
return strings.Replace(s, "-", "_", -1)
}

func filterRepos(repos []repository, typeName string) []repository {
filtered := make([]repository, 0)
func filterRepos(repos []rpmmd.RepoConfig, typeName string) []rpmmd.RepoConfig {
filtered := make([]rpmmd.RepoConfig, 0)
for _, repo := range repos {
if len(repo.ImageTypeTags) == 0 {
filtered = append(filtered, repo)
Expand Down Expand Up @@ -360,13 +281,12 @@ func main() {

// get repositories
repos := filterRepos(darm[distroName][archName], imgTypeName)
rpmmdRepos := convertRepos(repos)
if len(repos) == 0 {
fail(fmt.Sprintf("no repositories defined for %s/%s\n", distroName, archName))
}

fmt.Printf("Generating manifest for %s: ", config.Name)
mf, err := makeManifest(imgType, config, distribution, rpmmdRepos, archName, seedArg, rpmCacheRoot)
mf, err := makeManifest(imgType, config, distribution, repos, archName, seedArg, rpmCacheRoot)
if err != nil {
check(err)
}
Expand Down
123 changes: 20 additions & 103 deletions cmd/gen-manifests/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,55 +40,19 @@ func (mv *multiValue) Set(v string) error {
return nil
}

type repository struct {
Name string `json:"name"`
Id string `json:"id,omitempty"`
BaseURL string `json:"baseurl,omitempty"`
Metalink string `json:"metalink,omitempty"`
MirrorList string `json:"mirrorlist,omitempty"`
GPGKey string `json:"gpgkey,omitempty"`
CheckGPG bool `json:"check_gpg,omitempty"`
CheckRepoGPG bool `json:"check_repo_gpg,omitempty"`
IgnoreSSL bool `json:"ignore_ssl,omitempty"`
RHSM bool `json:"rhsm,omitempty"`
MetadataExpire string `json:"metadata_expire,omitempty"`
ImageTypeTags []string `json:"image_type_tags,omitempty"`
PackageSets []string `json:"package-sets,omitempty"`
}

type ostreeOptions struct {
Ref string `json:"ref"`
URL string `json:"url"`
Parent string `json:"parent"`
RHSM bool `json:"rhsm"`
}

type crBlueprint struct {
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
Version string `json:"version,omitempty"`
Packages []blueprint.Package `json:"packages,omitempty"`
Modules []blueprint.Package `json:"modules,omitempty"`
Groups []blueprint.Group `json:"groups,omitempty"`
Containers []blueprint.Container `json:"containers,omitempty"`
Customizations *blueprint.Customizations `json:"customizations,omitempty"`
Distro string `json:"distro,omitempty"`
Minimal bool `json:"minimal,omitempty"`
}

type buildRequest struct {
Distro string `json:"distro,omitempty"`
Arch string `json:"arch,omitempty"`
ImageType string `json:"image-type,omitempty"`
Repositories []repository `json:"repositories,omitempty"`
Config *BuildConfig `json:"config"`
Distro string `json:"distro,omitempty"`
Arch string `json:"arch,omitempty"`
ImageType string `json:"image-type,omitempty"`
Repositories []rpmmd.RepoConfig `json:"repositories,omitempty"`
Config *BuildConfig `json:"config"`
}

type BuildConfig struct {
Name string `json:"name"`
OSTree *ostreeOptions `json:"ostree,omitempty"`
Blueprint *crBlueprint `json:"blueprint,omitempty"`
Depends BuildDependency `json:"depends,omitempty"`
Name string `json:"name"`
OSTree *ostree.ImageOptions `json:"ostree,omitempty"`
Blueprint *blueprint.Blueprint `json:"blueprint,omitempty"`
Depends BuildDependency `json:"depends,omitempty"`
}

type BuildDependency struct {
Expand Down Expand Up @@ -232,7 +196,7 @@ func makeManifestJob(
imgType distro.ImageType,
bc BuildConfig,
distribution distro.Distro,
repos []repository,
repos []rpmmd.RepoConfig,
archName string,
seedArg int64,
path string,
Expand All @@ -245,13 +209,11 @@ func makeManifestJob(
cacheDir := filepath.Join(cacheRoot, archName+distribution.Name())

options := distro.ImageOptions{Size: 0}
if bc.OSTree != nil {
options.OSTree = &ostree.ImageOptions{
URL: bc.OSTree.URL,
ImageRef: bc.OSTree.Ref,
ParentRef: bc.OSTree.Parent,
RHSM: bc.OSTree.RHSM,
}
options.OSTree = bc.OSTree

var bp blueprint.Blueprint
if bc.Blueprint != nil {
bp = *bc.Blueprint
}

// add RHSM fact to detect changes
Expand All @@ -268,13 +230,8 @@ func makeManifestJob(
msgq <- msg
}()
msgq <- fmt.Sprintf("Starting job %s", filename)
rpmrepos := convertRepos(repos)
var bp blueprint.Blueprint
if bc.Blueprint != nil {
bp = blueprint.Blueprint(*bc.Blueprint)
}

manifest, _, err := imgType.Manifest(&bp, options, rpmrepos, seedArg)
manifest, _, err := imgType.Manifest(&bp, options, repos, seedArg)
if err != nil {
err = fmt.Errorf("[%s] failed: %s", filename, err)
return
Expand All @@ -291,10 +248,6 @@ func makeManifestJob(
err = fmt.Errorf("[%s] nil package specs", filename)
return
}

if bc.Blueprint != nil {
bp = blueprint.Blueprint(*bc.Blueprint)
}
} else {
packageSpecs = mockDepsolve(manifest.GetPackageSetChains())
}
Expand Down Expand Up @@ -337,43 +290,7 @@ func makeManifestJob(
return job
}

type DistroArchRepoMap map[string]map[string][]repository

func convertRepo(r repository) rpmmd.RepoConfig {
var urls []string
if r.BaseURL != "" {
urls = []string{r.BaseURL}
}

var keys []string
if r.GPGKey != "" {
keys = []string{r.GPGKey}
}

return rpmmd.RepoConfig{
Id: r.Id,
Name: r.Name,
BaseURLs: urls,
Metalink: r.Metalink,
MirrorList: r.MirrorList,
GPGKeys: keys,
CheckGPG: &r.CheckGPG,
CheckRepoGPG: &r.CheckRepoGPG,
IgnoreSSL: &r.IgnoreSSL,
MetadataExpire: r.MetadataExpire,
RHSM: r.RHSM,
ImageTypeTags: r.ImageTypeTags,
PackageSets: r.PackageSets,
}
}

func convertRepos(rr []repository) []rpmmd.RepoConfig {
cr := make([]rpmmd.RepoConfig, len(rr))
for idx, r := range rr {
cr[idx] = convertRepo(r)
}
return cr
}
type DistroArchRepoMap map[string]map[string][]rpmmd.RepoConfig

func readRepos() DistroArchRepoMap {
reposDir := "./test/data/repositories/"
Expand All @@ -397,7 +314,7 @@ func readRepos() DistroArchRepoMap {
if err != nil {
panic(err)
}
repos := make(map[string][]repository)
repos := make(map[string][]rpmmd.RepoConfig)
if err := json.Unmarshal(data, &repos); err != nil {
panic(err)
}
Expand Down Expand Up @@ -564,8 +481,8 @@ func save(ms manifest.OSBuildManifest, pkgs map[string][]rpmmd.PackageSpec, cont
return nil
}

func filterRepos(repos []repository, typeName string) []repository {
filtered := make([]repository, 0)
func filterRepos(repos []rpmmd.RepoConfig, typeName string) []rpmmd.RepoConfig {
filtered := make([]rpmmd.RepoConfig, 0)
for _, repo := range repos {
if len(repo.ImageTypeTags) == 0 {
filtered = append(filtered, repo)
Expand Down
10 changes: 6 additions & 4 deletions pkg/distro/fedora/images.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,7 @@ func iotImage(workload workload.Workload,
if err != nil {
return nil, fmt.Errorf("%s: %s", t.Name(), err.Error())
}
img := image.NewOSTreeDiskImage(commit)
img := image.NewOSTreeDiskImageFromCommit(commit)

distro := t.Arch().Distro()

Expand Down Expand Up @@ -522,9 +522,10 @@ func iotImage(workload workload.Workload,
Name: "fedora-iot",
}
img.OSName = "fedora-iot"
img.LockRoot = true

if !common.VersionLessThan(distro.Releasever(), "38") {
img.Ignition = true
img.KernelOptionsAppend = append(img.KernelOptionsAppend, "coreos.no_persist_ip")
supakeen marked this conversation as resolved.
Show resolved Hide resolved
switch img.Platform.GetImageFormat() {
case platform.FORMAT_RAW:
img.IgnitionPlatform = "metal"
Expand Down Expand Up @@ -565,7 +566,7 @@ func iotSimplifiedInstallerImage(workload workload.Workload,
if err != nil {
return nil, fmt.Errorf("%s: %s", t.Name(), err.Error())
}
rawImg := image.NewOSTreeDiskImage(commit)
rawImg := image.NewOSTreeDiskImageFromCommit(commit)

customizations := bp.Customizations
rawImg.Users = users.UsersFromBP(customizations.GetUsers())
Expand All @@ -585,10 +586,11 @@ func iotSimplifiedInstallerImage(workload workload.Workload,
Name: "fedora-iot",
}
rawImg.OSName = "fedora"
rawImg.LockRoot = true

if !common.VersionLessThan(t.arch.distro.osVersion, "38") {
rawImg.Ignition = true
rawImg.IgnitionPlatform = "metal"
rawImg.KernelOptionsAppend = append(rawImg.KernelOptionsAppend, "coreos.no_persist_ip")
if bpIgnition := customizations.GetIgnition(); bpIgnition != nil && bpIgnition.FirstBoot != nil && bpIgnition.FirstBoot.ProvisioningURL != "" {
rawImg.KernelOptionsAppend = append(rawImg.KernelOptionsAppend, "ignition.config.url="+bpIgnition.FirstBoot.ProvisioningURL)
}
Expand Down
Loading
Loading