From c91f3d7d2a29baefd358a32d34cf2151f4d73cd2 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 11 Apr 2024 14:51:04 +0200 Subject: [PATCH 01/18] Schutzfile: update osbuild dependency --- Schutzfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Schutzfile b/Schutzfile index c47bd7efa2..bac977665a 100644 --- a/Schutzfile +++ b/Schutzfile @@ -8,7 +8,7 @@ "fedora-39": { "dependencies": { "osbuild": { - "commit": "6549bf1992b9731d52df5416584fab3f014a421f" + "commit": "f26e62b23f547fc5ad665cb08cc52e6431657779" } }, "repos": [ From fd53a6b243111b0b463045281796e335f3824666 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 9 Apr 2024 22:47:55 +0200 Subject: [PATCH 02/18] dnfjson: osbuild-depsolve-dnf property rename Some repo properties were renamed in osbuild-depsolve-dnf to match the equivalent ones in dnf (libdnf). Of note is the SSLVerify property which was previously called IgnoreSSL and had the opposite effect. The nil value for SSLVerify falls back to 'true' in osbuild-depsolve-dnf which maintains the same behaviour. We make the new SSLVerify property a pointer so that we can express the unset/undefined/nil value properly and let osbuild-depsolve-dnf define the fallback value. This change breaks backwards compatibility. --- pkg/dnfjson/dnfjson.go | 13 +++++++------ pkg/dnfjson/dnfjson_test.go | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index 23b4c060f1..e239a56fec 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/osbuild/images/internal/common" "github.com/osbuild/images/pkg/rhsm" "github.com/osbuild/images/pkg/rpmmd" ) @@ -309,15 +310,15 @@ func (s *Solver) reposFromRPMMD(rpmRepos []rpmmd.RepoConfig) ([]repoConfig, erro } if rr.CheckGPG != nil { - dr.CheckGPG = *rr.CheckGPG + dr.GPGCheck = *rr.CheckGPG } if rr.CheckRepoGPG != nil { - dr.CheckRepoGPG = *rr.CheckRepoGPG + dr.RepoGPGCheck = *rr.CheckRepoGPG } if rr.IgnoreSSL != nil { - dr.IgnoreSSL = *rr.IgnoreSSL + dr.SSLVerify = common.ToPtr(!*rr.IgnoreSSL) } if rr.RHSM { @@ -347,9 +348,9 @@ type repoConfig struct { Metalink string `json:"metalink,omitempty"` MirrorList string `json:"mirrorlist,omitempty"` GPGKeys []string `json:"gpgkeys,omitempty"` - CheckGPG bool `json:"gpgcheck"` - CheckRepoGPG bool `json:"check_repogpg"` - IgnoreSSL bool `json:"ignoressl"` + GPGCheck bool `json:"gpgcheck"` + RepoGPGCheck bool `json:"repo_gpgcheck"` + SSLVerify *bool `json:"sslverify,omitempty"` SSLCACert string `json:"sslcacert,omitempty"` SSLClientKey string `json:"sslclientkey,omitempty"` SSLClientCert string `json:"sslclientcert,omitempty"` diff --git a/pkg/dnfjson/dnfjson_test.go b/pkg/dnfjson/dnfjson_test.go index ff88a6e495..e094815d29 100644 --- a/pkg/dnfjson/dnfjson_test.go +++ b/pkg/dnfjson/dnfjson_test.go @@ -699,5 +699,5 @@ func TestRepoConfigMarshalAlsmostEmpty(t *testing.T) { repoCfg := &repoConfig{} js, _ := json.Marshal(repoCfg) // double check here that anything that uses pointers has "omitempty" set - assert.Equal(t, string(js), `{"id":"","gpgcheck":false,"check_repogpg":false,"ignoressl":false}`) + assert.Equal(t, string(js), `{"id":"","gpgcheck":false,"repo_gpgcheck":false}`) } From d58952870d9d0ec74d0ba8cc26d004e5f8706d9c Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 3 Apr 2024 17:56:47 +0200 Subject: [PATCH 03/18] dnfjson: new osbuild-depsolve-dnf response format A new version of osbuild-depsolve-dnf returns, for the depsolve command, an object with two keys: - packages: a list of resolved packages which is the same as the old response format. - repos: a map (id -> config) of repositories used to resolve the packages. This map contains both repositories defined in the requests and repositories loaded from a directory, which is a new feature that osbuild-depsolve-dnf supports. Currently, the response handler ignores the repos part of the response and only returns the package list, so this has no functional effect outside the dnfjson package. See https://github.com/osbuild/osbuild/pull/1674 --- pkg/dnfjson/dnfjson.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index e239a56fec..8d6daeba7a 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -187,12 +187,14 @@ func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, erro } s.cache.updateInfo() - var result packageSpecs - if err := json.Unmarshal(output, &result); err != nil { + var result depsolveResult + dec := json.NewDecoder(bytes.NewReader(output)) + dec.DisallowUnknownFields() + if err := dec.Decode(&result); err != nil { return nil, err } - return result.toRPMMD(repoMap), nil + return result.Packages.toRPMMD(repoMap), nil } // FetchMetadata returns the list of all the available packages in repos and @@ -592,6 +594,11 @@ type transactionArgs struct { type packageSpecs []PackageSpec +type depsolveResult struct { + Packages packageSpecs `json:"packages"` + Repos map[string]repoConfig `json:"repos"` +} + // Package specification type PackageSpec struct { Name string `json:"name"` From 25a92fb6c32ab19e5ec57c79d0b3c50f20527b37 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 20 Mar 2024 16:21:45 +0100 Subject: [PATCH 04/18] dnfjson: add root_dir and releasever to request Add the new root_dir argument and releaever value to the depsolve request. The releasever is (currently) only required when root_dir is set, but it will eventually be required for all requests so let's set it always. --- pkg/dnfjson/dnfjson.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index 8d6daeba7a..f3e935897d 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -140,6 +140,8 @@ type Solver struct { // for each distribution. distro string + rootDir string + subscriptions *rhsm.Subscriptions } @@ -149,6 +151,13 @@ func NewSolver(modulePlatformID, releaseVer, arch, distro, cacheDir string) *Sol return s.NewWithConfig(modulePlatformID, releaseVer, arch, distro) } +// SetRootDir sets a path from which repository configurations, gpg keys, and +// vars are loaded during depsolve, instead of (or in addition to) the +// repositories and keys included in each depsolve request. +func (s *Solver) SetRootDir(path string) { + s.rootDir = path +} + // GetCacheDir returns a distro specific rpm cache directory // It ensures that the distro name is below the root cache directory, and if there is // a problem it returns the root cache instead of an error. @@ -432,6 +441,7 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[ } args := arguments{ Repos: dnfRepoMap, + RootDir: s.rootDir, Transactions: transactions, } @@ -439,6 +449,7 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[ Command: "depsolve", ModulePlatformID: s.modulePlatformID, Arch: s.arch, + Releasever: s.releaseVer, CacheDir: s.GetCacheDir(), Arguments: args, } @@ -456,6 +467,7 @@ func (s *Solver) makeDumpRequest(repos []rpmmd.RepoConfig) (*Request, error) { Command: "dump", ModulePlatformID: s.modulePlatformID, Arch: s.arch, + Releasever: s.releaseVer, CacheDir: s.GetCacheDir(), Arguments: arguments{ Repos: dnfRepos, @@ -475,6 +487,7 @@ func (s *Solver) makeSearchRequest(repos []rpmmd.RepoConfig, packages []string) ModulePlatformID: s.modulePlatformID, Arch: s.arch, CacheDir: s.GetCacheDir(), + Releasever: s.releaseVer, Arguments: arguments{ Repos: dnfRepos, Search: searchArgs{ @@ -528,6 +541,9 @@ type Request struct { // Platform ID, e.g., "platform:el8" ModulePlatformID string `json:"module_platform_id"` + // Distro Releasever, e.e., "8" + Releasever string `json:"releasever"` + // System architecture Arch string `json:"arch"` @@ -566,6 +582,10 @@ type arguments struct { // Depsolve package sets and repository mappings for this request Transactions []transactionArgs `json:"transactions"` + + // Load repository configurations, gpg keys, and vars from an os-root-like + // tree. + RootDir string `json:"root_dir"` } type searchArgs struct { From 2285a65ffc625880222d559f5d60a018ae4affd8 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 3 Apr 2024 21:21:43 +0200 Subject: [PATCH 05/18] dnfjson: fix makeDepsolveRequest() docstring The docstring was describing an older implementation of the function. --- pkg/dnfjson/dnfjson.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index f3e935897d..fbd2ade5c4 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -378,13 +378,11 @@ func (r *repoConfig) Hash() string { return r.repoHash } -// Helper function for creating a depsolve request payload. -// The request defines a sequence of transactions, each depsolving one of the -// elements of `pkgSets` in the order they appear. The `repoConfigs` are used -// as the base repositories for all transactions. The extra repository configs -// in `pkgsetsRepos` are used for each of the `pkgSets` with matching index. -// The length of `pkgsetsRepos` must match the length of `pkgSets` or be empty -// (nil or empty slice). +// Helper function for creating a depsolve request payload. The request defines +// a sequence of transactions, each depsolving one of the elements of `pkgSets` +// in the order they appear. The repositories are collected in the request +// arguments indexed by their ID, and each transaction lists the repositories +// it will use for depsolving. // // NOTE: Due to implementation limitations of DNF and dnf-json, each package set // in the chain must use all of the repositories used by its predecessor. From b7144e25fdf9d1c8fab9b6909aeb2b42c34fb9c3 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 3 Apr 2024 22:03:04 +0200 Subject: [PATCH 06/18] dnfjson: move the toRPMMD() conversion method to depsolveResult Now that the response contains packages and repository configurations we can do the conversion to rpmmd.PackageSet on the request itself directly without needing extra repo configs. This commit doesn't complete the move but simply moves the method to the depsolve response. --- pkg/dnfjson/dnfjson.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index fbd2ade5c4..b671b8614a 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -203,7 +203,7 @@ func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, erro return nil, err } - return result.Packages.toRPMMD(repoMap), nil + return result.toRPMMD(repoMap), nil } // FetchMetadata returns the list of all the available packages in repos and @@ -498,7 +498,8 @@ func (s *Solver) makeSearchRequest(repos []rpmmd.RepoConfig, packages []string) // convert internal a list of PackageSpecs to the rpmmd equivalent and attach // key and subscription information based on the repository configs. -func (pkgs packageSpecs) toRPMMD(repos map[string]rpmmd.RepoConfig) []rpmmd.PackageSpec { +func (result depsolveResult) toRPMMD(repos map[string]rpmmd.RepoConfig) []rpmmd.PackageSpec { + pkgs := result.Packages rpmDependencies := make([]rpmmd.PackageSpec, len(pkgs)) for i, dep := range pkgs { repo, ok := repos[dep.RepoID] From bc456eaf55f16302d3bd9a4560f62d1f3d6af44c Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 3 Apr 2024 22:10:35 +0200 Subject: [PATCH 07/18] dnfjson: use repository configs from response to convert packages Use the repository configurations that osbuild-depsolve-dnf returns to populate extra repo-related metadata on each package (like CheckGPG). This will now also include repositories loaded from the RootDir, so packages are properly configured even when they're retrieved from repositories not configured in the request. RHSM is treated as a special property now, outside of other repository properties and options. RHSM can only be set on repos defined in the request, not from repositories in the RootDir, so we keep a set of repository IDs that specify RHSM in order to set them on each package that requires it. --- pkg/dnfjson/dnfjson.go | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index b671b8614a..e602ce4f9c 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -175,7 +175,7 @@ func (s *Solver) GetCacheDir() string { // transactions in a chain. It returns a list of all packages (with solved // dependencies) that will be installed into the system. func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, error) { - req, repoMap, err := s.makeDepsolveRequest(pkgSets) + req, rhsmMap, err := s.makeDepsolveRequest(pkgSets) if err != nil { return nil, err } @@ -190,7 +190,7 @@ func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, erro } // touch repos to now now := time.Now().Local() - for _, r := range repoMap { + for _, r := range req.Arguments.Repos { // ignore errors _ = s.cache.touchRepo(r.Hash(), now) } @@ -203,7 +203,7 @@ func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, erro return nil, err } - return result.toRPMMD(repoMap), nil + return result.toRPMMD(rhsmMap), nil } // FetchMetadata returns the list of all the available packages in repos and @@ -384,22 +384,25 @@ func (r *repoConfig) Hash() string { // arguments indexed by their ID, and each transaction lists the repositories // it will use for depsolving. // +// The second return value is a map of repository IDs that have RHSM enabled. +// The RHSM property is not part of the dnf repository configuration so it's +// returned separately for setting the value on each package that requires it. +// // NOTE: Due to implementation limitations of DNF and dnf-json, each package set // in the chain must use all of the repositories used by its predecessor. // An error is returned if this requirement is not met. -func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[string]rpmmd.RepoConfig, error) { - +func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[string]bool, error) { // dedupe repository configurations but maintain order // the order in which repositories are added to the request affects the // order of the dependencies in the result repos := make([]rpmmd.RepoConfig, 0) - rpmRepoMap := make(map[string]rpmmd.RepoConfig) + rhsmMap := make(map[string]bool) for _, ps := range pkgSets { for _, repo := range ps.Repositories { id := repo.Hash() - if _, ok := rpmRepoMap[id]; !ok { - rpmRepoMap[id] = repo + if _, ok := rhsmMap[id]; !ok { + rhsmMap[id] = repo.RHSM repos = append(repos, repo) } } @@ -452,7 +455,7 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[ Arguments: args, } - return &req, rpmRepoMap, nil + return &req, rhsmMap, nil } // Helper function for creating a dump request payload @@ -498,8 +501,9 @@ func (s *Solver) makeSearchRequest(repos []rpmmd.RepoConfig, packages []string) // convert internal a list of PackageSpecs to the rpmmd equivalent and attach // key and subscription information based on the repository configs. -func (result depsolveResult) toRPMMD(repos map[string]rpmmd.RepoConfig) []rpmmd.PackageSpec { +func (result depsolveResult) toRPMMD(rhsm map[string]bool) []rpmmd.PackageSpec { pkgs := result.Packages + repos := result.Repos rpmDependencies := make([]rpmmd.PackageSpec, len(pkgs)) for i, dep := range pkgs { repo, ok := repos[dep.RepoID] @@ -514,16 +518,14 @@ func (result depsolveResult) toRPMMD(repos map[string]rpmmd.RepoConfig) []rpmmd. rpmDependencies[i].Arch = dep.Arch rpmDependencies[i].RemoteLocation = dep.RemoteLocation rpmDependencies[i].Checksum = dep.Checksum - if repo.CheckGPG != nil { - rpmDependencies[i].CheckGPG = *repo.CheckGPG - } - if repo.IgnoreSSL != nil { - rpmDependencies[i].IgnoreSSL = *repo.IgnoreSSL + rpmDependencies[i].CheckGPG = repo.GPGCheck + if verify := repo.SSLVerify; verify != nil { + rpmDependencies[i].IgnoreSSL = !*verify } // The ssl secrets will also be set if rhsm is true, // which should take priority. - if repo.RHSM { + if rhsm[dep.RepoID] { rpmDependencies[i].Secrets = "org.osbuild.rhsm" } else if repo.SSLClientKey != "" { rpmDependencies[i].Secrets = "org.osbuild.mtls" From 912c9b6329c98a1b8a6db255397eb7b2eeee7c41 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 3 Apr 2024 19:27:19 +0200 Subject: [PATCH 08/18] mocks/rpmrepo: write config in .repo format The WriteConfig() function wasn't being used. Repurpose it to write a .repo format config that we can use to test the new osbuild-depsolve-dnf feature. --- internal/mocks/rpmrepo/rpmrepo.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/internal/mocks/rpmrepo/rpmrepo.go b/internal/mocks/rpmrepo/rpmrepo.go index c51ae84919..9fdc5e679d 100644 --- a/internal/mocks/rpmrepo/rpmrepo.go +++ b/internal/mocks/rpmrepo/rpmrepo.go @@ -1,7 +1,7 @@ package rpmrepo import ( - "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" @@ -31,18 +31,32 @@ func (trs *testRepoServer) Close() { trs.Server.Close() } -// WriteConfig writes the repository config to the file defined by the given -// path. Assumes the location already exists. +// WriteConfig writes the repository config to the specified path in .repo +// format. Assumes the location already exists. func (trs *testRepoServer) WriteConfig(path string) { - fp, err := os.Create(path) - if err != nil { - panic(err) + cfgtmpl := `[%[1]s] +name=%[1]s +baseurl=%[2]s +gpgcheck=%[3]s +sslverify=%[4]s +` + + checkGPG := "0" + if trs.RepoConfig.CheckGPG != nil && *trs.RepoConfig.CheckGPG { + checkGPG = "1" } - data, err := json.Marshal(trs.RepoConfig) + sslverify := "1" + if trs.RepoConfig.IgnoreSSL != nil && *trs.RepoConfig.IgnoreSSL { + sslverify = "0" + } + + config := fmt.Sprintf(cfgtmpl, trs.RepoConfig.Name, trs.RepoConfig.BaseURLs[0], checkGPG, sslverify) + + fp, err := os.Create(path) if err != nil { panic(err) } - if _, err := fp.Write(data); err != nil { + if _, err := fp.Write([]byte(config)); err != nil { panic(err) } } From 7c7f282c7656f3f3b51f305d3292cf970471f08a Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 3 Apr 2024 19:51:38 +0200 Subject: [PATCH 09/18] dnfjson: convert TestDepsolver tests to iterable test cases --- pkg/dnfjson/dnfjson_test.go | 49 +++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/pkg/dnfjson/dnfjson_test.go b/pkg/dnfjson/dnfjson_test.go index e094815d29..cc5320fe81 100644 --- a/pkg/dnfjson/dnfjson_test.go +++ b/pkg/dnfjson/dnfjson_test.go @@ -26,33 +26,40 @@ func TestDepsolver(t *testing.T) { s := rpmrepo.NewTestServer() defer s.Close() - assert := assert.New(t) + type testCase struct { + packages [][]string + err error + } tmpdir := t.TempDir() solver := NewSolver("platform:el9", "9", "x86_64", "rhel9.0", tmpdir) - { // single depsolve - pkgsets := []rpmmd.PackageSet{{Include: []string{"kernel", "vim-minimal", "tmux", "zsh"}, Repositories: []rpmmd.RepoConfig{s.RepoConfig}, InstallWeakDeps: true}} // everything you'll ever need - - deps, err := solver.Depsolve(pkgsets) - if err != nil { - t.Fatal(err) - } - exp := expectedResult(s.RepoConfig) - assert.Equal(deps, exp) + testCases := map[string]testCase{ + "flat": { + packages: [][]string{{"kernel", "vim-minimal", "tmux", "zsh"}}, + err: nil, + }, + "chain": { + // chain depsolve of the same packages in order should produce the same result (at least in this case) + packages: [][]string{{"kernel"}, {"vim-minimal", "tmux", "zsh"}}, + err: nil, + }, } - { // chain depsolve of the same packages in order should produce the same result (at least in this case) - pkgsets := []rpmmd.PackageSet{ - {Include: []string{"kernel"}, Repositories: []rpmmd.RepoConfig{s.RepoConfig}, InstallWeakDeps: true}, - {Include: []string{"vim-minimal", "tmux", "zsh"}, Repositories: []rpmmd.RepoConfig{s.RepoConfig}}, - } - deps, err := solver.Depsolve(pkgsets) - if err != nil { - t.Fatal(err) - } - exp := expectedResult(s.RepoConfig) - assert.Equal(deps, exp) + for tcName := range testCases { + t.Run(tcName, func(t *testing.T) { + assert := assert.New(t) + tc := testCases[tcName] + pkgsets := make([]rpmmd.PackageSet, len(tc.packages)) + for idx := range tc.packages { + pkgsets[idx] = rpmmd.PackageSet{Include: tc.packages[idx], Repositories: []rpmmd.RepoConfig{s.RepoConfig}, InstallWeakDeps: true} + } + + deps, err := solver.Depsolve(pkgsets) + assert.Equal(tc.err, err) + exp := expectedResult(s.RepoConfig) + assert.Equal(deps, exp) + }) } } From 0943fdaf20509bd3426ea94ba8b086e82f8bb0fa Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 3 Apr 2024 20:00:33 +0200 Subject: [PATCH 10/18] dnfjson: add some error cases to the depsolver test --- pkg/dnfjson/dnfjson_test.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pkg/dnfjson/dnfjson_test.go b/pkg/dnfjson/dnfjson_test.go index cc5320fe81..01c8d42ca5 100644 --- a/pkg/dnfjson/dnfjson_test.go +++ b/pkg/dnfjson/dnfjson_test.go @@ -44,6 +44,18 @@ func TestDepsolver(t *testing.T) { packages: [][]string{{"kernel"}, {"vim-minimal", "tmux", "zsh"}}, err: nil, }, + "bad-flat": { + packages: [][]string{{"this-package-does-not-exist"}}, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "bad-chain": { + packages: [][]string{{"kernel"}, {"this-package-does-not-exist"}}, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "bad-chain-part-deux": { + packages: [][]string{{"this-package-does-not-exist"}, {"vim-minimal", "tmux", "zsh"}}, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, } for tcName := range testCases { @@ -57,8 +69,10 @@ func TestDepsolver(t *testing.T) { deps, err := solver.Depsolve(pkgsets) assert.Equal(tc.err, err) - exp := expectedResult(s.RepoConfig) - assert.Equal(deps, exp) + if err == nil { + exp := expectedResult(s.RepoConfig) + assert.Equal(deps, exp) + } }) } } From a3fcb9c10da1b4237b5c3d90a572c127af3e7efe Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 3 Apr 2024 22:49:26 +0200 Subject: [PATCH 11/18] dnfjson: add tests that use RootDir with the test repo metadata Add more test cases that use the test repo metadata but only define the repo through the RootDir. This test requires the new osbuild-depsolve-dnf that supports reading repositories from a directory. --- pkg/dnfjson/dnfjson_test.go | 43 ++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/pkg/dnfjson/dnfjson_test.go b/pkg/dnfjson/dnfjson_test.go index 01c8d42ca5..0658cbc5a3 100644 --- a/pkg/dnfjson/dnfjson_test.go +++ b/pkg/dnfjson/dnfjson_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "flag" "fmt" + "os" + "path/filepath" "strings" "testing" @@ -11,6 +13,7 @@ import ( "github.com/osbuild/images/internal/mocks/rpmrepo" "github.com/osbuild/images/pkg/rpmmd" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var forceDNF = flag.Bool("force-dnf", false, "force dnf testing, making them fail instead of skip if dnf isn't installed") @@ -28,32 +31,69 @@ func TestDepsolver(t *testing.T) { type testCase struct { packages [][]string + repos []rpmmd.RepoConfig + rootDir string err error } tmpdir := t.TempDir() solver := NewSolver("platform:el9", "9", "x86_64", "rhel9.0", tmpdir) + rootDir := t.TempDir() + reposDir := filepath.Join(rootDir, "etc", "yum.repos.d") + require.NoError(t, os.MkdirAll(reposDir, 0777)) + s.WriteConfig(filepath.Join(reposDir, "test.repo")) + testCases := map[string]testCase{ "flat": { packages: [][]string{{"kernel", "vim-minimal", "tmux", "zsh"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, err: nil, }, "chain": { // chain depsolve of the same packages in order should produce the same result (at least in this case) packages: [][]string{{"kernel"}, {"vim-minimal", "tmux", "zsh"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, err: nil, }, "bad-flat": { packages: [][]string{{"this-package-does-not-exist"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, }, "bad-chain": { packages: [][]string{{"kernel"}, {"this-package-does-not-exist"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, }, "bad-chain-part-deux": { packages: [][]string{{"this-package-does-not-exist"}, {"vim-minimal", "tmux", "zsh"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "flat+dir": { + packages: [][]string{{"kernel", "vim-minimal", "tmux", "zsh"}}, + rootDir: rootDir, + err: nil, + }, + "chain+dir": { + packages: [][]string{{"kernel"}, {"vim-minimal", "tmux", "zsh"}}, + rootDir: rootDir, + err: nil, + }, + "bad-flat+dir": { + packages: [][]string{{"this-package-does-not-exist"}}, + rootDir: rootDir, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "bad-chain+dir": { + packages: [][]string{{"kernel"}, {"this-package-does-not-exist"}}, + rootDir: rootDir, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "bad-chain-part-deux+dir": { + packages: [][]string{{"this-package-does-not-exist"}, {"vim-minimal", "tmux", "zsh"}}, + rootDir: rootDir, err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, }, } @@ -64,9 +104,10 @@ func TestDepsolver(t *testing.T) { tc := testCases[tcName] pkgsets := make([]rpmmd.PackageSet, len(tc.packages)) for idx := range tc.packages { - pkgsets[idx] = rpmmd.PackageSet{Include: tc.packages[idx], Repositories: []rpmmd.RepoConfig{s.RepoConfig}, InstallWeakDeps: true} + pkgsets[idx] = rpmmd.PackageSet{Include: tc.packages[idx], Repositories: tc.repos, InstallWeakDeps: true} } + solver.SetRootDir(tc.rootDir) deps, err := solver.Depsolve(pkgsets) assert.Equal(tc.err, err) if err == nil { From 841b2b15f3f3ddf057bdce3fb48ce379995ecd34 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 9 Apr 2024 19:42:49 +0200 Subject: [PATCH 12/18] dnfjson: return rpmmd.RepoConfig list from depsolve Convert the repos returned from osbuild-depsolve-dnf to rpmmd.RepoConfig and return them alongside the packages. This is needed to get access to GPG keys that were loaded from repositories read from a directory. --- cmd/build/main.go | 4 +-- cmd/gen-manifests/main.go | 4 +-- cmd/osbuild-playground/playground.go | 2 +- pkg/dnfjson/dnfjson.go | 46 ++++++++++++++++++++++------ pkg/dnfjson/dnfjson_test.go | 4 +-- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index 67c039e501..602826e07b 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -154,11 +154,11 @@ func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d dist solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) depsolvedSets := make(map[string][]rpmmd.PackageSpec) for name, pkgSet := range packageSets { - res, err := solver.Depsolve(pkgSet) + pkgs, _, err := solver.Depsolve(pkgSet) if err != nil { return nil, err } - depsolvedSets[name] = res + depsolvedSets[name] = pkgs } return depsolvedSets, nil } diff --git a/cmd/gen-manifests/main.go b/cmd/gen-manifests/main.go index bb76311ce5..c761466bbb 100644 --- a/cmd/gen-manifests/main.go +++ b/cmd/gen-manifests/main.go @@ -382,11 +382,11 @@ func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d dist solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) depsolvedSets := make(map[string][]rpmmd.PackageSpec) for name, pkgSet := range packageSets { - res, err := solver.Depsolve(pkgSet) + packages, _, err := solver.Depsolve(pkgSet) if err != nil { return nil, err } - depsolvedSets[name] = res + depsolvedSets[name] = packages } return depsolvedSets, nil } diff --git a/cmd/osbuild-playground/playground.go b/cmd/osbuild-playground/playground.go index 44a6b28c39..d082183049 100644 --- a/cmd/osbuild-playground/playground.go +++ b/cmd/osbuild-playground/playground.go @@ -36,7 +36,7 @@ func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos packageSpecs := make(map[string][]rpmmd.PackageSpec) for name, chain := range manifest.GetPackageSetChains() { - packages, err := solver.Depsolve(chain) + packages, _, err := solver.Depsolve(chain) if err != nil { panic(fmt.Sprintf("failed to depsolve for pipeline %s: %s\n", name, err.Error())) } diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index e602ce4f9c..705a8ab955 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -174,10 +174,10 @@ func (s *Solver) GetCacheDir() string { // their associated repositories. Each package set is depsolved as a separate // transactions in a chain. It returns a list of all packages (with solved // dependencies) that will be installed into the system. -func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, error) { +func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, []rpmmd.RepoConfig, error) { req, rhsmMap, err := s.makeDepsolveRequest(pkgSets) if err != nil { - return nil, err + return nil, nil, err } // get non-exclusive read lock @@ -186,7 +186,7 @@ func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, erro output, err := run(s.dnfJsonCmd, req) if err != nil { - return nil, err + return nil, nil, err } // touch repos to now now := time.Now().Local() @@ -200,10 +200,11 @@ func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, erro dec := json.NewDecoder(bytes.NewReader(output)) dec.DisallowUnknownFields() if err := dec.Decode(&result); err != nil { - return nil, err + return nil, nil, err } - return result.toRPMMD(rhsmMap), nil + packages, repos := result.toRPMMD(rhsmMap) + return packages, repos, nil } // FetchMetadata returns the list of all the available packages in repos and @@ -499,9 +500,10 @@ func (s *Solver) makeSearchRequest(repos []rpmmd.RepoConfig, packages []string) return &req, nil } -// convert internal a list of PackageSpecs to the rpmmd equivalent and attach -// key and subscription information based on the repository configs. -func (result depsolveResult) toRPMMD(rhsm map[string]bool) []rpmmd.PackageSpec { +// convert internal a list of PackageSpecs and map of repoConfig to the rpmmd +// equivalents and attach key and subscription information based on the +// repository configs. +func (result depsolveResult) toRPMMD(rhsm map[string]bool) ([]rpmmd.PackageSpec, []rpmmd.RepoConfig) { pkgs := result.Packages repos := result.Repos rpmDependencies := make([]rpmmd.PackageSpec, len(pkgs)) @@ -531,7 +533,33 @@ func (result depsolveResult) toRPMMD(rhsm map[string]bool) []rpmmd.PackageSpec { rpmDependencies[i].Secrets = "org.osbuild.mtls" } } - return rpmDependencies + + repoConfigs := make([]rpmmd.RepoConfig, 0, len(repos)) + for repoID := range repos { + repo := repos[repoID] + var ignoreSSL bool + if sslVerify := repo.SSLVerify; sslVerify != nil { + ignoreSSL = !*sslVerify + } + repoConfigs = append(repoConfigs, rpmmd.RepoConfig{ + Id: repo.ID, + Name: repo.Name, + BaseURLs: repo.BaseURLs, + Metalink: repo.Metalink, + MirrorList: repo.MirrorList, + GPGKeys: repo.GPGKeys, + CheckGPG: &repo.GPGCheck, + CheckRepoGPG: &repo.RepoGPGCheck, + IgnoreSSL: &ignoreSSL, + MetadataExpire: repo.MetadataExpire, + ModuleHotfixes: repo.ModuleHotfixes, + Enabled: common.ToPtr(true), + SSLCACert: repo.SSLCACert, + SSLClientKey: repo.SSLClientKey, + SSLClientCert: repo.SSLClientCert, + }) + } + return rpmDependencies, repoConfigs } // Request command and arguments for dnf-json diff --git a/pkg/dnfjson/dnfjson_test.go b/pkg/dnfjson/dnfjson_test.go index 0658cbc5a3..000dcaedf0 100644 --- a/pkg/dnfjson/dnfjson_test.go +++ b/pkg/dnfjson/dnfjson_test.go @@ -108,7 +108,7 @@ func TestDepsolver(t *testing.T) { } solver.SetRootDir(tc.rootDir) - deps, err := solver.Depsolve(pkgsets) + deps, _, err := solver.Depsolve(pkgsets) assert.Equal(tc.err, err) if err == nil { exp := expectedResult(s.RepoConfig) @@ -699,7 +699,7 @@ func TestErrorRepoInfo(t *testing.T) { solver := NewSolver("f38", "38", "x86_64", "fedora-38", "/tmp/cache") for idx, tc := range testCases { t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { - _, err := solver.Depsolve([]rpmmd.PackageSet{ + _, _, err := solver.Depsolve([]rpmmd.PackageSet{ { Include: []string{"osbuild"}, Exclude: nil, From 741b4ff05ab70cf53097b94eb0252487f79d8b96 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 9 Apr 2024 20:13:17 +0200 Subject: [PATCH 13/18] cmd: return repo map from depsolve() Return the repository configurations from the depsolve() functions used in the gen-manifests and build commands. These repository configurations are a map keyed by the pipeline name they belong to, just like package sets. We don't do anything with these yet, so assign them to _ to work around compile errors. --- cmd/build/main.go | 13 ++++++++----- cmd/gen-manifests/main.go | 23 +++++++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index 602826e07b..bd00592a29 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -82,13 +82,14 @@ func makeManifest(imgType distro.ImageType, config BuildConfig, distribution dis fmt.Fprintf(os.Stderr, "[WARNING]\n%s", strings.Join(warnings, "\n")) } - packageSpecs, err := depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) + packageSpecs, repoConfigs, err := depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) if err != nil { return nil, fmt.Errorf("[ERROR] depsolve failed: %s", err.Error()) } if packageSpecs == nil { return nil, fmt.Errorf("[ERROR] depsolve did not return any packages") } + _ = repoConfigs if config.Blueprint != nil { bp = blueprint.Blueprint(*config.Blueprint) @@ -150,17 +151,19 @@ func resolvePipelineCommits(commitSources map[string][]ostree.SourceSpec) (map[s return commits, nil } -func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) { +func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) for name, pkgSet := range packageSets { - pkgs, _, err := solver.Depsolve(pkgSet) + pkgs, repos, err := solver.Depsolve(pkgSet) if err != nil { - return nil, err + return nil, nil, err } depsolvedSets[name] = pkgs + repoSets[name] = repos } - return depsolvedSets, nil + return depsolvedSets, repoSets, nil } func save(ms manifest.OSBuildManifest, fpath string) error { diff --git a/cmd/gen-manifests/main.go b/cmd/gen-manifests/main.go index c761466bbb..e7a7512485 100644 --- a/cmd/gen-manifests/main.go +++ b/cmd/gen-manifests/main.go @@ -240,8 +240,9 @@ func makeManifestJob( } var packageSpecs map[string][]rpmmd.PackageSpec + var repoConfigs map[string][]rpmmd.RepoConfig if content["packages"] { - packageSpecs, err = depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) + packageSpecs, repoConfigs, err = depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) if err != nil { err = fmt.Errorf("[%s] depsolve failed: %s", filename, err.Error()) return @@ -251,8 +252,9 @@ func makeManifestJob( return } } else { - packageSpecs = mockDepsolve(manifest.GetPackageSetChains()) + packageSpecs, repoConfigs = mockDepsolve(manifest.GetPackageSetChains(), repos) } + _ = repoConfigs var containerSpecs map[string][]container.Spec if content["containers"] { @@ -378,21 +380,24 @@ func mockResolveCommits(commitSources map[string][]ostree.SourceSpec) map[string return commits } -func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) { +func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) for name, pkgSet := range packageSets { - packages, _, err := solver.Depsolve(pkgSet) + packages, repos, err := solver.Depsolve(pkgSet) if err != nil { - return nil, err + return nil, nil, err } depsolvedSets[name] = packages + repoSets[name] = repos } - return depsolvedSets, nil + return depsolvedSets, repoSets, nil } -func mockDepsolve(packageSets map[string][]rpmmd.PackageSet) map[string][]rpmmd.PackageSpec { +func mockDepsolve(packageSets map[string][]rpmmd.PackageSet, repos []rpmmd.RepoConfig) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig) { depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) for name, pkgSetChain := range packageSets { specSet := make([]rpmmd.PackageSpec, 0) for _, pkgSet := range pkgSetChain { @@ -411,8 +416,10 @@ func mockDepsolve(packageSets map[string][]rpmmd.PackageSet) map[string][]rpmmd. } } depsolvedSets[name] = specSet + repoSets[name] = repos } - return depsolvedSets + + return depsolvedSets, repoSets } func save(ms manifest.OSBuildManifest, pkgs map[string][]rpmmd.PackageSpec, containers map[string][]container.Spec, commits map[string][]ostree.CommitSpec, cr buildRequest, path, filename string, metadata bool) error { From bdbfbf15a15ba56dddea66012f1278eeb6944973 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 10 Apr 2024 00:26:10 +0200 Subject: [PATCH 14/18] manifest: use new repo configs when serializing manifests Pass down the repository configurations we get from osbuild-depsolve-dnf into the manifest serialization functions. These repositories are used to add gpg keys to import for each rpm stage in a pipeline. The repository configurations are appended to the ones already defined in each pipeline before serialization. This will duplicate repositories in each pipeline, effectively duplicating gpg key importing. This wont have any effect on the build or the final image. --- cmd/build/main.go | 2 +- cmd/gen-manifests/main.go | 2 +- cmd/osbuild-playground/playground.go | 2 +- pkg/distro/distro_test.go | 4 ++- pkg/image/bootc_disk_test.go | 2 +- pkg/manifest/anaconda_installer.go | 3 ++- pkg/manifest/anaconda_installer_iso_tree.go | 2 +- .../anaconda_installer_iso_tree_test.go | 26 +++++++++---------- pkg/manifest/build.go | 5 ++-- pkg/manifest/build_test.go | 2 +- pkg/manifest/commit_server_tree.go | 3 ++- pkg/manifest/coreos_installer.go | 3 ++- pkg/manifest/empty.go | 5 +++- pkg/manifest/manifest.go | 4 +-- pkg/manifest/os.go | 4 ++- pkg/manifest/os_test.go | 2 +- pkg/manifest/ostree_deployment.go | 2 +- pkg/manifest/pipeline.go | 4 +-- pkg/manifest/raw_bootc.go | 2 +- pkg/manifest/raw_bootc_export_test.go | 4 +-- pkg/manifest/raw_bootc_test.go | 6 ++--- 21 files changed, 50 insertions(+), 39 deletions(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index bd00592a29..1184c59f6b 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -105,7 +105,7 @@ func makeManifest(imgType distro.ImageType, config BuildConfig, distribution dis return nil, fmt.Errorf("[ERROR] ostree commit resolution failed: %s\n", err.Error()) } - mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs) + mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, nil) if err != nil { return nil, fmt.Errorf("[ERROR] manifest serialization failed: %s", err.Error()) } diff --git a/cmd/gen-manifests/main.go b/cmd/gen-manifests/main.go index e7a7512485..58cbe2462e 100644 --- a/cmd/gen-manifests/main.go +++ b/cmd/gen-manifests/main.go @@ -276,7 +276,7 @@ func makeManifestJob( commitSpecs = mockResolveCommits(manifest.GetOSTreeSourceSpecs()) } - mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs) + mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs) if err != nil { return fmt.Errorf("[%s] manifest serialization failed: %s", filename, err.Error()) } diff --git a/cmd/osbuild-playground/playground.go b/cmd/osbuild-playground/playground.go index d082183049..ba6dab04ed 100644 --- a/cmd/osbuild-playground/playground.go +++ b/cmd/osbuild-playground/playground.go @@ -48,7 +48,7 @@ func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos fmt.Fprintf(os.Stderr, "could not clean dnf cache: %s", err.Error()) } - bytes, err := manifest.Serialize(packageSpecs, nil, nil) + bytes, err := manifest.Serialize(packageSpecs, nil, nil, nil) if err != nil { panic("failed to serialize manifest: " + err.Error()) } diff --git a/pkg/distro/distro_test.go b/pkg/distro/distro_test.go index 772685d6e7..631810f944 100644 --- a/pkg/distro/distro_test.go +++ b/pkg/distro/distro_test.go @@ -150,8 +150,10 @@ func TestImageTypePipelineNames(t *testing.T) { } packageSets := make(map[string][]rpmmd.PackageSpec, len(allPipelines)) + repoSets := make(map[string][]rpmmd.RepoConfig, len(allPipelines)) for _, plName := range allPipelines { packageSets[plName] = minimalPackageSet + repoSets[plName] = repos } m, _, err := imageType.Manifest(&bp, options, repos, seed) @@ -172,7 +174,7 @@ func TestImageTypePipelineNames(t *testing.T) { } commits[name] = commitSpecs } - mf, err := m.Serialize(packageSets, containers, commits) + mf, err := m.Serialize(packageSets, containers, commits, repoSets) assert.NoError(err) pm := new(manifest) err = json.Unmarshal(mf, pm) diff --git a/pkg/image/bootc_disk_test.go b/pkg/image/bootc_disk_test.go index 7f8bdc1c68..81a0f3eca4 100644 --- a/pkg/image/bootc_disk_test.go +++ b/pkg/image/bootc_disk_test.go @@ -81,7 +81,7 @@ func makeBootcDiskImageOsbuildManifest(t *testing.T, opts *bootcDiskImageTestOpt "image": []container.Spec{{Source: "other-src", Digest: makeFakeDigest(t), ImageID: makeFakeDigest(t)}}, } - osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil) + osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil, nil) require.Nil(t, err) return osbuildManifest diff --git a/pkg/manifest/anaconda_installer.go b/pkg/manifest/anaconda_installer.go index 8342316f1f..204fdf571f 100644 --- a/pkg/manifest/anaconda_installer.go +++ b/pkg/manifest/anaconda_installer.go @@ -174,7 +174,7 @@ func (p *AnacondaInstaller) getPackageSpecs() []rpmmd.PackageSpec { return p.packageSpecs } -func (p *AnacondaInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) { +func (p *AnacondaInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } @@ -182,6 +182,7 @@ func (p *AnacondaInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []con if p.kernelName != "" { p.kernelVer = rpmmd.GetVerStrFromPackageSpecListPanic(p.packageSpecs, p.kernelName) } + p.repos = append(p.repos, rpmRepos...) } func (p *AnacondaInstaller) serializeEnd() { diff --git a/pkg/manifest/anaconda_installer_iso_tree.go b/pkg/manifest/anaconda_installer_iso_tree.go index ccd3f390bf..2d6f53cb86 100644 --- a/pkg/manifest/anaconda_installer_iso_tree.go +++ b/pkg/manifest/anaconda_installer_iso_tree.go @@ -156,7 +156,7 @@ func (p *AnacondaInstallerISOTree) getBuildPackages(_ Distro) []string { return packages } -func (p *AnacondaInstallerISOTree) serializeStart(_ []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) { +func (p *AnacondaInstallerISOTree) serializeStart(_ []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec, _ []rpmmd.RepoConfig) { if p.ostreeCommitSpec != nil || p.containerSpec != nil { panic("double call to serializeStart()") } diff --git a/pkg/manifest/anaconda_installer_iso_tree_test.go b/pkg/manifest/anaconda_installer_iso_tree_test.go index 6bebc43489..8d4055a12c 100644 --- a/pkg/manifest/anaconda_installer_iso_tree_test.go +++ b/pkg/manifest/anaconda_installer_iso_tree_test.go @@ -192,11 +192,11 @@ func TestAnacondaISOTreePayloadsBad(t *testing.T) { assert.PanicsWithValue( "pipeline supports at most one ostree commit", - func() { pipeline.serializeStart(nil, nil, make([]ostree.CommitSpec, 2)) }, + func() { pipeline.serializeStart(nil, nil, make([]ostree.CommitSpec, 2), nil) }, ) assert.PanicsWithValue( "pipeline supports at most one container", - func() { pipeline.serializeStart(nil, make([]container.Spec, 2), nil) }, + func() { pipeline.serializeStart(nil, make([]container.Spec, 2), nil, nil) }, ) } @@ -216,7 +216,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { t.Run("plain", func(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.OSPipeline = osPayload - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, payloadStages, @@ -229,7 +229,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.OSPipeline = osPayload pipeline.KSPath = testKsPath - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.kickstart"), @@ -242,7 +242,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { pipeline.OSPipeline = osPayload pipeline.KSPath = testKsPath pipeline.ISOLinux = true - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux", "org.osbuild.kickstart"), @@ -255,7 +255,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { pipeline.KSPath = testKsPath pipeline.ISOLinux = true pipeline.UnattendedKickstart = true - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux", "org.osbuild.kickstart"), @@ -270,7 +270,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { pipeline.ISOLinux = true pipeline.UnattendedKickstart = true pipeline.NoPasswd = []string{`%wheel`, `%sudo`} - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux", "org.osbuild.kickstart"), @@ -302,7 +302,7 @@ func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { t.Run("plain", func(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.KSPath = testKsPath - pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}) + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, payloadStages, @@ -314,7 +314,7 @@ func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.KSPath = testKsPath pipeline.ISOLinux = true - pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}) + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux"), variantStages)) @@ -325,7 +325,7 @@ func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { pipeline.KSPath = testKsPath pipeline.ISOLinux = true pipeline.UnattendedKickstart = true - pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}) + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux"), variantStages)) @@ -338,7 +338,7 @@ func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { pipeline.ISOLinux = true pipeline.UnattendedKickstart = true pipeline.NoPasswd = []string{`%wheel`, `%sudo`} - pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}) + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux"), variantStages)) @@ -369,7 +369,7 @@ func TestAnacondaISOTreeSerializeWithContainer(t *testing.T) { t.Run("kspath", func(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.KSPath = testKsPath - pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil) + pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, payloadStages, @@ -381,7 +381,7 @@ func TestAnacondaISOTreeSerializeWithContainer(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.KSPath = testKsPath pipeline.ISOLinux = true - pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil) + pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux"), variantStages)) diff --git a/pkg/manifest/build.go b/pkg/manifest/build.go index ef82d47823..eca633d44a 100644 --- a/pkg/manifest/build.go +++ b/pkg/manifest/build.go @@ -99,11 +99,12 @@ func (p *BuildrootFromPackages) getPackageSpecs() []rpmmd.PackageSpec { return p.packageSpecs } -func (p *BuildrootFromPackages) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) { +func (p *BuildrootFromPackages) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } p.packageSpecs = packages + p.repos = append(p.repos, rpmRepos...) } func (p *BuildrootFromPackages) serializeEnd() { @@ -198,7 +199,7 @@ func (p *BuildrootFromContainer) getContainerSpecs() []container.Spec { return p.containerSpecs } -func (p *BuildrootFromContainer) serializeStart(_ []rpmmd.PackageSpec, containerSpecs []container.Spec, _ []ostree.CommitSpec) { +func (p *BuildrootFromContainer) serializeStart(_ []rpmmd.PackageSpec, containerSpecs []container.Spec, _ []ostree.CommitSpec, _ []rpmmd.RepoConfig) { if len(p.containerSpecs) > 0 { panic("double call to serializeStart()") } diff --git a/pkg/manifest/build_test.go b/pkg/manifest/build_test.go index ddfea70054..eb1da5680a 100644 --- a/pkg/manifest/build_test.go +++ b/pkg/manifest/build_test.go @@ -109,7 +109,7 @@ func TestNewBuildFromContainerSpecs(t *testing.T) { } // containerSpecs is "nil" until serializeStart populates it require.Nil(t, build.getContainerSpecs()) - build.serializeStart(nil, fakeContainerSpecs, nil) + build.serializeStart(nil, fakeContainerSpecs, nil, nil) assert.Equal(t, build.getContainerSpecs(), fakeContainerSpecs) osbuildPipeline := build.serialize() diff --git a/pkg/manifest/commit_server_tree.go b/pkg/manifest/commit_server_tree.go index b5233a3c0b..8f9c8669f0 100644 --- a/pkg/manifest/commit_server_tree.go +++ b/pkg/manifest/commit_server_tree.go @@ -80,11 +80,12 @@ func (p *OSTreeCommitServer) getPackageSpecs() []rpmmd.PackageSpec { return p.packageSpecs } -func (p *OSTreeCommitServer) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) { +func (p *OSTreeCommitServer) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } p.packageSpecs = packages + p.repos = append(p.repos, rpmRepos...) } func (p *OSTreeCommitServer) serializeEnd() { diff --git a/pkg/manifest/coreos_installer.go b/pkg/manifest/coreos_installer.go index a38f15aef0..26e208d0f7 100644 --- a/pkg/manifest/coreos_installer.go +++ b/pkg/manifest/coreos_installer.go @@ -136,7 +136,7 @@ func (p *CoreOSInstaller) getPackageSpecs() []rpmmd.PackageSpec { return p.packageSpecs } -func (p *CoreOSInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) { +func (p *CoreOSInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } @@ -144,6 +144,7 @@ func (p *CoreOSInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []conta if p.kernelName != "" { p.kernelVer = rpmmd.GetVerStrFromPackageSpecListPanic(p.packageSpecs, p.kernelName) } + p.repos = append(p.repos, rpmRepos...) } func (p *CoreOSInstaller) getInline() []string { diff --git a/pkg/manifest/empty.go b/pkg/manifest/empty.go index 8338e9d039..05c60cb4e1 100644 --- a/pkg/manifest/empty.go +++ b/pkg/manifest/empty.go @@ -22,6 +22,8 @@ type ContentTest struct { containerSpecs []container.Spec commitSpecs []ostree.CommitSpec + repos []rpmmd.RepoConfig + // serialization flag serializing bool } @@ -63,13 +65,14 @@ func (p *ContentTest) getOSTreeCommits() []ostree.CommitSpec { return p.commitSpecs } -func (p *ContentTest) serializeStart(pkgs []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) { +func (p *ContentTest) serializeStart(pkgs []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if p.serializing { panic("double call to serializeStart()") } p.packageSpecs = pkgs p.containerSpecs = containers p.commitSpecs = commits + p.repos = rpmRepos p.serializing = true } diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index 5a1f508a79..1ae10c5a0f 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -138,14 +138,14 @@ func (m Manifest) GetOSTreeSourceSpecs() map[string][]ostree.SourceSpec { return ostreeSpecs } -func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec) (OSBuildManifest, error) { +func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, rpmRepos map[string][]rpmmd.RepoConfig) (OSBuildManifest, error) { pipelines := make([]osbuild.Pipeline, 0) packages := make([]rpmmd.PackageSpec, 0) commits := make([]ostree.CommitSpec, 0) inline := make([]string, 0) containers := make([]container.Spec, 0) for _, pipeline := range m.pipelines { - pipeline.serializeStart(packageSets[pipeline.Name()], containerSpecs[pipeline.Name()], ostreeCommits[pipeline.Name()]) + pipeline.serializeStart(packageSets[pipeline.Name()], containerSpecs[pipeline.Name()], ostreeCommits[pipeline.Name()], rpmRepos[pipeline.Name()]) } for _, pipeline := range m.pipelines { commits = append(commits, pipeline.getOSTreeCommits()...) diff --git a/pkg/manifest/os.go b/pkg/manifest/os.go index b605abbb39..52f1549553 100644 --- a/pkg/manifest/os.go +++ b/pkg/manifest/os.go @@ -338,7 +338,7 @@ func (p *OS) getContainerSpecs() []container.Spec { return p.containerSpecs } -func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) { +func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } @@ -355,6 +355,8 @@ func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container if p.KernelName != "" { p.kernelVer = rpmmd.GetVerStrFromPackageSpecListPanic(p.packageSpecs, p.KernelName) } + + p.repos = append(p.repos, rpmRepos...) } func (p *OS) serializeEnd() { diff --git a/pkg/manifest/os_test.go b/pkg/manifest/os_test.go index 87924a5e70..c6bd63e1cb 100644 --- a/pkg/manifest/os_test.go +++ b/pkg/manifest/os_test.go @@ -30,7 +30,7 @@ func NewTestOS() *OS { packages := []rpmmd.PackageSpec{ {Name: "pkg1", Checksum: "sha1:c02524e2bd19490f2a7167958f792262754c5f46"}, } - os.serializeStart(packages, nil, nil) + os.serializeStart(packages, nil, nil, repos) return os } diff --git a/pkg/manifest/ostree_deployment.go b/pkg/manifest/ostree_deployment.go index 40dbef5efa..a7518aa97a 100644 --- a/pkg/manifest/ostree_deployment.go +++ b/pkg/manifest/ostree_deployment.go @@ -161,7 +161,7 @@ func (p *OSTreeDeployment) getContainerSources() []container.SourceSpec { } } -func (p *OSTreeDeployment) serializeStart(packages []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) { +func (p *OSTreeDeployment) serializeStart(_ []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec, _ []rpmmd.RepoConfig) { if p.ostreeSpec != nil || p.containerSpec != nil { panic("double call to serializeStart()") } diff --git a/pkg/manifest/pipeline.go b/pkg/manifest/pipeline.go index ed1d67e79b..a9325693e6 100644 --- a/pkg/manifest/pipeline.go +++ b/pkg/manifest/pipeline.go @@ -53,7 +53,7 @@ type Pipeline interface { // its full Spec. See the ostree package for more details. getOSTreeCommitSources() []ostree.SourceSpec - serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec) + serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec, []rpmmd.RepoConfig) serializeEnd() serialize() osbuild.Pipeline @@ -166,7 +166,7 @@ func NewBase(name string, build Build) Base { // serializeStart must be called exactly once before each call // to serialize(). -func (p Base) serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec) { +func (p Base) serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec, []rpmmd.RepoConfig) { } // serializeEnd must be called exactly once after each call to diff --git a/pkg/manifest/raw_bootc.go b/pkg/manifest/raw_bootc.go index dfe4c2da75..1f9c93d823 100644 --- a/pkg/manifest/raw_bootc.go +++ b/pkg/manifest/raw_bootc.go @@ -66,7 +66,7 @@ func (p *RawBootcImage) getContainerSpecs() []container.Spec { return p.containerSpecs } -func (p *RawBootcImage) serializeStart(_ []rpmmd.PackageSpec, containerSpecs []container.Spec, _ []ostree.CommitSpec) { +func (p *RawBootcImage) serializeStart(_ []rpmmd.PackageSpec, containerSpecs []container.Spec, _ []ostree.CommitSpec, _ []rpmmd.RepoConfig) { if len(p.containerSpecs) > 0 { panic("double call to serializeStart()") } diff --git a/pkg/manifest/raw_bootc_export_test.go b/pkg/manifest/raw_bootc_export_test.go index 450bedd34d..2e87203c8a 100644 --- a/pkg/manifest/raw_bootc_export_test.go +++ b/pkg/manifest/raw_bootc_export_test.go @@ -15,6 +15,6 @@ func (rbc *RawBootcImage) Serialize() osbuild.Pipeline { return rbc.serialize() } -func (rbc *RawBootcImage) SerializeStart(a []rpmmd.PackageSpec, b []container.Spec, c []ostree.CommitSpec) { - rbc.serializeStart(a, b, c) +func (rbc *RawBootcImage) SerializeStart(a []rpmmd.PackageSpec, b []container.Spec, c []ostree.CommitSpec, r []rpmmd.RepoConfig) { + rbc.serializeStart(a, b, c, r) } diff --git a/pkg/manifest/raw_bootc_test.go b/pkg/manifest/raw_bootc_test.go index d144654ebb..94b94f8abb 100644 --- a/pkg/manifest/raw_bootc_test.go +++ b/pkg/manifest/raw_bootc_test.go @@ -51,7 +51,7 @@ func TestRawBootcImageSerialize(t *testing.T) { rawBootcPipeline.Users = []users.User{{Name: "root", Key: common.ToPtr("some-ssh-key")}} rawBootcPipeline.KernelOptionsAppend = []string{"karg1", "karg2"} - rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil) + rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil, nil) imagePipeline := rawBootcPipeline.Serialize() assert.Equal(t, "image", imagePipeline.Name) @@ -70,7 +70,7 @@ func TestRawBootcImageSerializeMountsValidated(t *testing.T) { rawBootcPipeline := manifest.NewRawBootcImage(build, nil, nil) // note that we create a partition table without /boot here rawBootcPipeline.PartitionTable = testdisk.MakeFakePartitionTable("/", "/missing-boot") - rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil) + rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil, nil) assert.PanicsWithError(t, `required mounts for bootupd stage [/boot /boot/efi] missing`, func() { rawBootcPipeline.Serialize() }) @@ -83,7 +83,7 @@ func TestRawBootcImageSerializeValidatesUsers(t *testing.T) { rawBootcPipeline := manifest.NewRawBootcImage(build, nil, nil) rawBootcPipeline.PartitionTable = testdisk.MakeFakePartitionTable("/", "/boot", "/boot/efi") - rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil) + rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil, nil) for _, tc := range []struct { users []users.User From 99332b928aed17a55befef74a7c92fc6e970d19c Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 11 Apr 2024 15:20:01 +0200 Subject: [PATCH 15/18] github: install pinned osbuild commit for unit tests Run the setup-osbuild-repo script before running unit tests in github actions to get the pinned osbuild-depsolve-dnf rpm. Some steps had to be reordered for this. The repo setup requires the repository to be checked out and the dependency installation should happen after the osbuild repo setup. --- .github/workflows/tests.yml | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 64f00f9af8..31cc452e6c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,17 +20,20 @@ jobs: image: registry.fedoraproject.org/fedora:latest steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up repository for pinned osbuild commit + run: ./test/scripts/setup-osbuild-repo + # krb5-devel is needed to test internal/upload/koji package # gcc is needed to build the mock depsolver binary for the unit tests # gpgme-devel is needed for container upload dependencies - name: Install build and test dependencies run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf btrfs-progs-devel device-mapper-devel - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Mark the working directory as safe for git run: git config --global --add safe.directory "$(pwd)" @@ -55,14 +58,17 @@ jobs: - name: Enable crb repo run: dnf config-manager --set-enabled crb - - name: Install build and test dependencies - run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf device-mapper-devel - - name: Check out code into the Go module directory uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Set up repository for pinned osbuild commit + run: ./test/scripts/setup-osbuild-repo + + - name: Install build and test dependencies + run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf device-mapper-devel + - name: Mark the working directory as safe for git run: git config --global --add safe.directory "$(pwd)" @@ -87,14 +93,19 @@ jobs: - name: Enable powertools repo run: dnf config-manager --set-enabled powertools - - name: Install build and test dependencies - run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf device-mapper-devel - - name: Check out code into the Go module directory uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Set up repository for pinned osbuild commit + run: | + dnf -y install python3 + ./test/scripts/setup-osbuild-repo + + - name: Install build and test dependencies + run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf device-mapper-devel + - name: Mark the working directory as safe for git run: git config --global --add safe.directory "$(pwd)" From 56eebc3c29ef9c0d4249b8e266884beb656362df Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 11 Apr 2024 15:21:01 +0200 Subject: [PATCH 16/18] Schutzfile: add osbuild commits for centos-8 and centos-9 Needed for unit tests in github actions. --- Schutzfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Schutzfile b/Schutzfile index bac977665a..b7f0f4b550 100644 --- a/Schutzfile +++ b/Schutzfile @@ -5,6 +5,20 @@ "ref": "quay.io/centos-bootc/bootc-image-builder@sha256:3a75eeb703b4f8a711f43f37bafb2c3e84f5dd12b499c8146603f05be8cccabd" } }, + "centos-8": { + "dependencies": { + "osbuild": { + "commit": "f26e62b23f547fc5ad665cb08cc52e6431657779" + } + } + }, + "centos-9": { + "dependencies": { + "osbuild": { + "commit": "f26e62b23f547fc5ad665cb08cc52e6431657779" + } + } + }, "fedora-39": { "dependencies": { "osbuild": { From ea6e95ab67d09b29a6d15f95bfba63019bad7759 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 11 Apr 2024 15:21:30 +0200 Subject: [PATCH 17/18] test/imgtestlib: remove use of match statement We run setup-osbuild-repo on CentOS Stream 8 now which imports the library but doesn't have a new enough python version to include the 'match' statement. --- test/scripts/imgtestlib.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/scripts/imgtestlib.py b/test/scripts/imgtestlib.py index 0df01847b2..9ad6545347 100644 --- a/test/scripts/imgtestlib.py +++ b/test/scripts/imgtestlib.py @@ -432,12 +432,10 @@ def rng_seed_env(): def host_container_arch(): host_arch = os.uname().machine - match host_arch: - case "x86_64": - return "amd64" - case "aarch64": - return "arm64" - return host_arch + return { + "x86_64": "amd64", + "aarch64": "arm64" + }.get(host_arch, host_arch) def is_manifest_list(data): From 6acadfba1845893e3eb23073abc3e983075a344f Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 11 Apr 2024 16:24:05 +0200 Subject: [PATCH 18/18] test/configure-generators: setup-osbuild-repo for inspect Since we now inspect the generated manifests in he generate-manifests jobs [1], we need the pinned version of osbuild. [1] cdbf17dd8e3a22510f06b5c0839c1bc6bc918f98 --- .gitlab-ci.yml | 35 +++++++++++++++++++++++++++++++++ test/scripts/generate-gitlab-ci | 1 + 2 files changed, 36 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6e4adca8f..da11e853c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2401,6 +2401,7 @@ generate-manifests-centos-10-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros centos-10 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2419,6 +2420,7 @@ generate-manifests-centos-10-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros centos-10 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2437,6 +2439,7 @@ generate-manifests-centos-8-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros centos-8 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2455,6 +2458,7 @@ generate-manifests-centos-9-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros centos-9 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2473,6 +2477,7 @@ generate-manifests-centos-9-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros centos-9 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2491,6 +2496,7 @@ generate-manifests-fedora-38-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros fedora-38 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2509,6 +2515,7 @@ generate-manifests-fedora-38-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros fedora-38 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2527,6 +2534,7 @@ generate-manifests-fedora-39-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros fedora-39 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2545,6 +2553,7 @@ generate-manifests-fedora-39-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros fedora-39 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2563,6 +2572,7 @@ generate-manifests-rhel-10.0-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-10.0 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2581,6 +2591,7 @@ generate-manifests-rhel-10.0-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-10.0 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2599,6 +2610,7 @@ generate-manifests-rhel-8.10-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.10 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2617,6 +2629,7 @@ generate-manifests-rhel-8.10-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.10 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2635,6 +2648,7 @@ generate-manifests-rhel-8.4-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.4 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2653,6 +2667,7 @@ generate-manifests-rhel-8.4-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.4 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2671,6 +2686,7 @@ generate-manifests-rhel-8.5-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.5 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2689,6 +2705,7 @@ generate-manifests-rhel-8.5-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.5 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2707,6 +2724,7 @@ generate-manifests-rhel-8.6-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.6 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2725,6 +2743,7 @@ generate-manifests-rhel-8.6-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.6 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2743,6 +2762,7 @@ generate-manifests-rhel-8.7-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.7 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2761,6 +2781,7 @@ generate-manifests-rhel-8.7-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.7 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2779,6 +2800,7 @@ generate-manifests-rhel-8.8-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.8 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2797,6 +2819,7 @@ generate-manifests-rhel-8.8-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.8 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2815,6 +2838,7 @@ generate-manifests-rhel-8.9-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.9 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2833,6 +2857,7 @@ generate-manifests-rhel-8.9-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.9 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2851,6 +2876,7 @@ generate-manifests-rhel-9.0-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.0 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2869,6 +2895,7 @@ generate-manifests-rhel-9.0-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.0 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2887,6 +2914,7 @@ generate-manifests-rhel-9.1-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.1 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2905,6 +2933,7 @@ generate-manifests-rhel-9.1-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.1 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2923,6 +2952,7 @@ generate-manifests-rhel-9.2-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.2 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2941,6 +2971,7 @@ generate-manifests-rhel-9.2-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.2 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2959,6 +2990,7 @@ generate-manifests-rhel-9.3-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.3 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2977,6 +3009,7 @@ generate-manifests-rhel-9.3-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.3 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2995,6 +3028,7 @@ generate-manifests-rhel-9.4-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.4 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -3013,6 +3047,7 @@ generate-manifests-rhel-9.4-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.4 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do diff --git a/test/scripts/generate-gitlab-ci b/test/scripts/generate-gitlab-ci index 979edd78cc..c5144e40b0 100755 --- a/test/scripts/generate-gitlab-ci +++ b/test/scripts/generate-gitlab-ci @@ -139,6 +139,7 @@ generate-manifests-{distro}-{arch}: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches {arch} --distros {distro} --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do