Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cmd/build: use the new manifestgen package #1165

Merged
merged 3 commits into from
Jan 29, 2025
Merged

cmd/build: use the new manifestgen package #1165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c392883
manifestgen: add support for repos override
mvo5 Jan 21, 2025
7074474
manifestgen: add new `manifestgen.Options.WarningsOutput`
mvo5 Jan 21, 2025
8d70047
cmd/build: use the new manifestgen package
mvo5 Jan 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 50 additions & 136 deletions cmd/build/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
package main

import (
"encoding/json"
"bytes"
"flag"
"fmt"
"os"
Expand All @@ -13,139 +13,18 @@ import (
"github.com/osbuild/images/internal/cmdutil"
"github.com/osbuild/images/pkg/arch"
"github.com/osbuild/images/pkg/blueprint"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/distrofactory"
"github.com/osbuild/images/pkg/manifest"
"github.com/osbuild/images/pkg/manifestgen"
"github.com/osbuild/images/pkg/osbuild"
"github.com/osbuild/images/pkg/reporegistry"
"github.com/osbuild/images/pkg/rhsm/facts"
"github.com/osbuild/images/pkg/rpmmd"
)

func makeManifest(
config *buildconfig.BuildConfig,
imgType distro.ImageType,
distribution distro.Distro,
repos []rpmmd.RepoConfig,
archName string,
cacheRoot string,
) (manifest.OSBuildManifest, error) {
cacheDir := filepath.Join(cacheRoot, archName+distribution.Name())

options := config.Options

// add RHSM fact to detect changes
options.Facts = &facts.ImageOptions{
APIType: facts.TEST_APITYPE,
}

var bp blueprint.Blueprint
if config.Blueprint != nil {
bp = blueprint.Blueprint(*config.Blueprint)
}
seedArg, err := cmdutil.SeedArgFor(config, imgType.Name(), distribution.Name(), archName)
if err != nil {
return nil, err
}

manifest, warnings, err := imgType.Manifest(&bp, options, repos, &seedArg)
if err != nil {
return nil, fmt.Errorf("[ERROR] manifest generation failed: %w", err)
}
if len(warnings) > 0 {
fmt.Fprintf(os.Stderr, "[WARNING]\n%s", strings.Join(warnings, "\n"))
}

depsolvedSets, err := manifestgen.DefaultDepsolver(cacheDir, manifest.GetPackageSetChains(), distribution, archName)
if err != nil {
return nil, fmt.Errorf("[ERROR] depsolve failed: %w", err)
}
if depsolvedSets == nil {
return nil, fmt.Errorf("[ERROR] depsolve did not return any packages")
}

if config.Blueprint != nil {
bp = blueprint.Blueprint(*config.Blueprint)
}

containerSpecs, err := manifestgen.DefaultContainerResolver(manifest.GetContainerSourceSpecs(), archName)
if err != nil {
return nil, fmt.Errorf("[ERROR] container resolution failed: %w", err)
}

commitSpecs, err := manifestgen.DefaultCommitResolver(manifest.GetOSTreeSourceSpecs())
if err != nil {
return nil, fmt.Errorf("[ERROR] ostree commit resolution failed: %w", err)
}

mf, err := manifest.Serialize(depsolvedSets, containerSpecs, commitSpecs, nil)
if err != nil {
return nil, fmt.Errorf("[ERROR] manifest serialization failed: %w", err)
}

return mf, nil
}

func save(ms manifest.OSBuildManifest, fpath string) error {
b, err := json.MarshalIndent(ms, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal data for %q: %w", fpath, err)
}
b = append(b, '\n') // add new line at end of file
fp, err := os.Create(fpath)
if err != nil {
return fmt.Errorf("failed to create output file %q: %w", fpath, err)
}
defer fp.Close()
if _, err := fp.Write(b); err != nil {
return fmt.Errorf("failed to write output file %q: %w", fpath, err)
}
return nil
}

func u(s string) string {
return strings.Replace(s, "-", "_", -1)
}

// loadRepos loads the repositories defined at the path and returns just the
// ones for the specified architecture. If the path is a directory, the distro
// name is appended to the path as a filename (with .json extension).
func loadRepos(path, distro, arch string) ([]rpmmd.RepoConfig, error) {
pstat, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("failed to stat %q: %w", path, err)
}

if pstat.IsDir() {
path = filepath.Join(path, fmt.Sprintf("%s.json", distro))
}

repos, err := rpmmd.LoadRepositoriesFromFile(path)
if err != nil {
return nil, fmt.Errorf("failed to load repositories from %q: %w", path, err)
}

// NOTE: this can be empty
return repos[arch], nil
}

func filterRepos(repos []rpmmd.RepoConfig, typeName string) []rpmmd.RepoConfig {
filtered := make([]rpmmd.RepoConfig, 0)
for _, repo := range repos {
if len(repo.ImageTypeTags) == 0 {
filtered = append(filtered, repo)
} else {
for _, tt := range repo.ImageTypeTags {
if tt == typeName {
filtered = append(filtered, repo)
break
}
}
}
}
return filtered
}

func run() error {
// common args
var outputDir, osbuildStore, rpmCacheRoot, repositories string
Expand All @@ -172,7 +51,6 @@ func run() error {
}

distroFac := distrofactory.NewDefault()

config, err := buildconfig.New(configFile)
if err != nil {
return err
Expand Down Expand Up @@ -204,32 +82,68 @@ func run() error {
return fmt.Errorf("invalid image type %q for distro %q and arch %q: %w", imgTypeName, distroName, archName, err)
}

// get repositories
repos, err := loadRepos(repositories, distroName, archName)
if err != nil {
return fmt.Errorf("failed to get repositories for %s/%s: %w", distroName, archName, err)
var reporeg *reporegistry.RepoRegistry
var overrideRepos []rpmmd.RepoConfig
if st, err := os.Stat(repositories); err == nil && !st.IsDir() {
// anything that is not a dir is tried to be loaded as a file
// to allow "-repositories <arbitrarily-named-file>.json"
repoConfig, err := rpmmd.LoadRepositoriesFromFile(repositories)
if err != nil {
return fmt.Errorf("failed to load repositories from %q: %w", repositories, err)
}
overrideRepos = repoConfig[archName]
} else {
// HACK: reporegistry hardcodes adding the "repositories"
// dir to the "repoConfigPaths" but the interface of
// "images" does not expect this.
// XXX: should we fix this in reporegistry?
thozza marked this conversation as resolved.
Show resolved Hide resolved
repositories = filepath.Join(repositories, "..")
reporeg, err = reporegistry.New([]string{repositories})
if err != nil {
return fmt.Errorf("failed to load repositories from %q: %w", repositories, err)
}
}
repos = filterRepos(repos, imgTypeName)
if len(repos) == 0 {
return fmt.Errorf("no repositories defined for %s/%s", distroName, archName)
seedArg, err := cmdutil.SeedArgFor(config, imgType.Name(), distribution.Name(), archName)
if err != nil {
return err
}

fmt.Printf("Generating manifest for %s: ", config.Name)
mf, err := makeManifest(config, imgType, distribution, repos, archName, rpmCacheRoot)
var mf bytes.Buffer
manifestOpts := manifestgen.Options{
Output: &mf,
Cachedir: filepath.Join(rpmCacheRoot, archName+distribution.Name()),
WarningsOutput: os.Stderr,
OverrideRepos: overrideRepos,
CustomSeed: &seedArg,
}
// add RHSM fact to detect changes
config.Options.Facts = &facts.ImageOptions{
APIType: facts.TEST_APITYPE,
}
if config.Blueprint == nil {
config.Blueprint = &blueprint.Blueprint{}
}

mg, err := manifestgen.New(reporeg, &manifestOpts)
if err != nil {
return err
return fmt.Errorf("[ERROR] manifest generator creation failed: %w", err)
}
if err := mg.Generate(config.Blueprint, distribution, imgType, arch, &config.Options); err != nil {
return fmt.Errorf("[ERROR] manifest generation failed: %w", err)
}
fmt.Print("DONE\n")

manifestPath := filepath.Join(buildDir, "manifest.json")
if err := save(mf, manifestPath); err != nil {
return err
// nolint:gosec
if err := os.WriteFile(manifestPath, mf.Bytes(), 0644); err != nil {
mvo5 marked this conversation as resolved.
Show resolved Hide resolved
return fmt.Errorf("failed to write output file %q: %w", manifestPath, err)
}

fmt.Printf("Building manifest: %s\n", manifestPath)

jobOutput := filepath.Join(outputDir, buildName)
_, err = osbuild.RunOSBuild(mf, osbuildStore, jobOutput, imgType.Exports(), checkpoints, nil, false, os.Stderr)
_, err = osbuild.RunOSBuild(mf.Bytes(), osbuildStore, jobOutput, imgType.Exports(), checkpoints, nil, false, os.Stderr)
if err != nil {
return err
}
Expand Down
37 changes: 29 additions & 8 deletions pkg/manifestgen/manifestgen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,19 @@ type Options struct {
// content can be read
SBOMWriter SBOMWriterFunc

// WarningsOutput will receive any warnings that are part of
// the manifest generation. If it is unset any warnings will
// generate an error.
WarningsOutput io.Writer

// CustomSeed overrides the default rng seed, this is mostly
// useful for testing
CustomSeed *int64

// OverrideRepos overrides the default repository selection.
// This is mostly useful for testing
OverrideRepos []rpmmd.RepoConfig

// Custom "solver" functions, if unset the defaults will be
// used. Only needed for specialized use-cases.
Depsolver DepsolveFunc
Expand All @@ -65,12 +74,14 @@ type Generator struct {
containerResolver ContainerResolverFunc
commitResolver CommitResolverFunc
sbomWriter SBOMWriterFunc
warningsOutput io.Writer

reporegistry *reporegistry.RepoRegistry

rpmDownloader osbuild.RpmDownloader

customSeed *int64
customSeed *int64
overrideRepos []rpmmd.RepoConfig
}

// New will create a new manifest generator
Expand All @@ -88,7 +99,9 @@ func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, er
commitResolver: opts.CommitResolver,
rpmDownloader: opts.RpmDownloader,
sbomWriter: opts.SBOMWriter,
warningsOutput: opts.WarningsOutput,
customSeed: opts.CustomSeed,
overrideRepos: opts.OverrideRepos,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nitpick: would it make sense to return an error in case opts.OverrideRepos and reporegistry are both provided?, Because in such case, the reporegistry will always be ignored. Similarly, would an error make sense if none is provided?

}
if mg.out == nil {
mg.out = os.Stdout
Expand All @@ -108,14 +121,19 @@ func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, er

// Generate will generate a new manifest for the given distro/imageType/arch
// combination.
func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgType distro.ImageType, a distro.Arch, imgOpts *distro.ImageOptions) error {
func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgType distro.ImageType, a distro.Arch, imgOpts *distro.ImageOptions) (err error) {
if imgOpts == nil {
imgOpts = &distro.ImageOptions{}
}

repos, err := mg.reporegistry.ReposByImageTypeName(dist.Name(), a.Name(), imgType.Name())
if err != nil {
return err
var repos []rpmmd.RepoConfig
if mg.overrideRepos != nil {
repos = mg.overrideRepos
} else {
repos, err = mg.reporegistry.ReposByImageTypeName(dist.Name(), a.Name(), imgType.Name())
if err != nil {
return err
}
}
// To support "user" a.k.a. "3rd party" repositories, these
// will have to be added to the repos with
Expand All @@ -126,9 +144,12 @@ func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgTy
return err
}
if len(warnings) > 0 {
// XXX: what can we do here? for things like json output?
// what are these warnings?
return fmt.Errorf("warnings during manifest creation: %v", strings.Join(warnings, "\n"))
warn := strings.Join(warnings, "\n")
if mg.warningsOutput != nil {
fmt.Fprint(mg.warningsOutput, warn)
} else {
return fmt.Errorf("Warnings during manifest creation:\n%v", warn)
}
}
depsolved, err := mg.depsolver(mg.cacheDir, preManifest.GetPackageSetChains(), dist, a.Name())
if err != nil {
Expand Down
51 changes: 47 additions & 4 deletions pkg/manifestgen/manifestgen_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,10 +144,11 @@ func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d
for _, pkgSet := range pkgSets {
for _, pkgName := range pkgSet.Include {
resolvedSet.Packages = append(resolvedSet.Packages, rpmmd.PackageSpec{
Name: pkgName,
Checksum: sha256For(pkgName),
Path: fmt.Sprintf("path/%s.rpm", pkgName),
RepoID: repoId,
Name: pkgName,
Checksum: sha256For(pkgName),
Path: fmt.Sprintf("path/%s.rpm", pkgName),
RepoID: repoId,
RemoteLocation: fmt.Sprintf("%s/%s.rpm", pkgSet.Repositories[0].BaseURLs[0], pkgName),
})
resolvedSet.Repos = append(resolvedSet.Repos, rpmmd.RepoConfig{
Id: repoId,
Expand Down Expand Up @@ -330,3 +331,45 @@ func TestManifestGeneratorSeed(t *testing.T) {
}
}
}

func TestManifestGeneratorOverrideRepos(t *testing.T) {
repos, err := testrepos.New()
assert.NoError(t, err)
fac := distrofactory.NewDefault()

filter, err := imagefilter.New(fac, repos)
assert.NoError(t, err)
res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64")
assert.NoError(t, err)
assert.Equal(t, 1, len(res))

for _, withOverrideRepos := range []bool{false, true} {
t.Run(fmt.Sprintf("withOverrideRepos: %v", withOverrideRepos), func(t *testing.T) {
var osbuildManifest bytes.Buffer
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
}
if withOverrideRepos {
opts.OverrideRepos = []rpmmd.RepoConfig{
{
Name: "overriden_repo",
BaseURLs: []string{"http://example.com/overriden-repo"},
},
}
}

mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)

var bp blueprint.Blueprint
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
assert.NoError(t, err)
if withOverrideRepos {
assert.Contains(t, osbuildManifest.String(), "http://example.com/overriden-repo/kernel.rpm")
} else {
assert.NotContains(t, osbuildManifest.String(), "http://example.com/overriden-repo/kernel.rpm")
}
})
}
}
Loading