diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 64f00f9af8..31cc452e6c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,17 +20,20 @@ jobs: image: registry.fedoraproject.org/fedora:latest steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up repository for pinned osbuild commit + run: ./test/scripts/setup-osbuild-repo + # krb5-devel is needed to test internal/upload/koji package # gcc is needed to build the mock depsolver binary for the unit tests # gpgme-devel is needed for container upload dependencies - name: Install build and test dependencies run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf btrfs-progs-devel device-mapper-devel - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Mark the working directory as safe for git run: git config --global --add safe.directory "$(pwd)" @@ -55,14 +58,17 @@ jobs: - name: Enable crb repo run: dnf config-manager --set-enabled crb - - name: Install build and test dependencies - run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf device-mapper-devel - - name: Check out code into the Go module directory uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Set up repository for pinned osbuild commit + run: ./test/scripts/setup-osbuild-repo + + - name: Install build and test dependencies + run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf device-mapper-devel + - name: Mark the working directory as safe for git run: git config --global --add safe.directory "$(pwd)" @@ -87,14 +93,19 @@ jobs: - name: Enable powertools repo run: dnf config-manager --set-enabled powertools - - name: Install build and test dependencies - run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf device-mapper-devel - - name: Check out code into the Go module directory uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Set up repository for pinned osbuild commit + run: | + dnf -y install python3 + ./test/scripts/setup-osbuild-repo + + - name: Install build and test dependencies + run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf device-mapper-devel + - name: Mark the working directory as safe for git run: git config --global --add safe.directory "$(pwd)" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6e4adca8f..da11e853c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2401,6 +2401,7 @@ generate-manifests-centos-10-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros centos-10 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2419,6 +2420,7 @@ generate-manifests-centos-10-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros centos-10 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2437,6 +2439,7 @@ generate-manifests-centos-8-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros centos-8 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2455,6 +2458,7 @@ generate-manifests-centos-9-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros centos-9 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2473,6 +2477,7 @@ generate-manifests-centos-9-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros centos-9 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2491,6 +2496,7 @@ generate-manifests-fedora-38-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros fedora-38 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2509,6 +2515,7 @@ generate-manifests-fedora-38-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros fedora-38 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2527,6 +2534,7 @@ generate-manifests-fedora-39-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros fedora-39 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2545,6 +2553,7 @@ generate-manifests-fedora-39-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros fedora-39 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2563,6 +2572,7 @@ generate-manifests-rhel-10.0-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-10.0 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2581,6 +2591,7 @@ generate-manifests-rhel-10.0-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-10.0 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2599,6 +2610,7 @@ generate-manifests-rhel-8.10-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.10 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2617,6 +2629,7 @@ generate-manifests-rhel-8.10-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.10 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2635,6 +2648,7 @@ generate-manifests-rhel-8.4-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.4 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2653,6 +2667,7 @@ generate-manifests-rhel-8.4-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.4 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2671,6 +2686,7 @@ generate-manifests-rhel-8.5-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.5 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2689,6 +2705,7 @@ generate-manifests-rhel-8.5-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.5 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2707,6 +2724,7 @@ generate-manifests-rhel-8.6-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.6 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2725,6 +2743,7 @@ generate-manifests-rhel-8.6-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.6 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2743,6 +2762,7 @@ generate-manifests-rhel-8.7-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.7 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2761,6 +2781,7 @@ generate-manifests-rhel-8.7-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.7 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2779,6 +2800,7 @@ generate-manifests-rhel-8.8-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.8 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2797,6 +2819,7 @@ generate-manifests-rhel-8.8-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.8 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2815,6 +2838,7 @@ generate-manifests-rhel-8.9-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.9 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2833,6 +2857,7 @@ generate-manifests-rhel-8.9-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-8.9 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2851,6 +2876,7 @@ generate-manifests-rhel-9.0-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.0 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2869,6 +2895,7 @@ generate-manifests-rhel-9.0-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.0 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2887,6 +2914,7 @@ generate-manifests-rhel-9.1-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.1 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2905,6 +2933,7 @@ generate-manifests-rhel-9.1-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.1 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2923,6 +2952,7 @@ generate-manifests-rhel-9.2-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.2 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2941,6 +2971,7 @@ generate-manifests-rhel-9.2-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.2 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2959,6 +2990,7 @@ generate-manifests-rhel-9.3-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.3 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2977,6 +3009,7 @@ generate-manifests-rhel-9.3-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.3 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -2995,6 +3028,7 @@ generate-manifests-rhel-9.4-ppc64le: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.4 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do @@ -3013,6 +3047,7 @@ generate-manifests-rhel-9.4-s390x: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches s390x --distros rhel-9.4 --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do diff --git a/Schutzfile b/Schutzfile index c47bd7efa2..b7f0f4b550 100644 --- a/Schutzfile +++ b/Schutzfile @@ -5,10 +5,24 @@ "ref": "quay.io/centos-bootc/bootc-image-builder@sha256:3a75eeb703b4f8a711f43f37bafb2c3e84f5dd12b499c8146603f05be8cccabd" } }, + "centos-8": { + "dependencies": { + "osbuild": { + "commit": "f26e62b23f547fc5ad665cb08cc52e6431657779" + } + } + }, + "centos-9": { + "dependencies": { + "osbuild": { + "commit": "f26e62b23f547fc5ad665cb08cc52e6431657779" + } + } + }, "fedora-39": { "dependencies": { "osbuild": { - "commit": "6549bf1992b9731d52df5416584fab3f014a421f" + "commit": "f26e62b23f547fc5ad665cb08cc52e6431657779" } }, "repos": [ diff --git a/cmd/build/main.go b/cmd/build/main.go index 67c039e501..1184c59f6b 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -82,13 +82,14 @@ func makeManifest(imgType distro.ImageType, config BuildConfig, distribution dis fmt.Fprintf(os.Stderr, "[WARNING]\n%s", strings.Join(warnings, "\n")) } - packageSpecs, err := depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) + packageSpecs, repoConfigs, err := depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) if err != nil { return nil, fmt.Errorf("[ERROR] depsolve failed: %s", err.Error()) } if packageSpecs == nil { return nil, fmt.Errorf("[ERROR] depsolve did not return any packages") } + _ = repoConfigs if config.Blueprint != nil { bp = blueprint.Blueprint(*config.Blueprint) @@ -104,7 +105,7 @@ func makeManifest(imgType distro.ImageType, config BuildConfig, distribution dis return nil, fmt.Errorf("[ERROR] ostree commit resolution failed: %s\n", err.Error()) } - mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs) + mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, nil) if err != nil { return nil, fmt.Errorf("[ERROR] manifest serialization failed: %s", err.Error()) } @@ -150,17 +151,19 @@ func resolvePipelineCommits(commitSources map[string][]ostree.SourceSpec) (map[s return commits, nil } -func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) { +func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) for name, pkgSet := range packageSets { - res, err := solver.Depsolve(pkgSet) + pkgs, repos, err := solver.Depsolve(pkgSet) if err != nil { - return nil, err + return nil, nil, err } - depsolvedSets[name] = res + depsolvedSets[name] = pkgs + repoSets[name] = repos } - return depsolvedSets, nil + return depsolvedSets, repoSets, nil } func save(ms manifest.OSBuildManifest, fpath string) error { diff --git a/cmd/gen-manifests/main.go b/cmd/gen-manifests/main.go index bb76311ce5..58cbe2462e 100644 --- a/cmd/gen-manifests/main.go +++ b/cmd/gen-manifests/main.go @@ -240,8 +240,9 @@ func makeManifestJob( } var packageSpecs map[string][]rpmmd.PackageSpec + var repoConfigs map[string][]rpmmd.RepoConfig if content["packages"] { - packageSpecs, err = depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) + packageSpecs, repoConfigs, err = depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) if err != nil { err = fmt.Errorf("[%s] depsolve failed: %s", filename, err.Error()) return @@ -251,8 +252,9 @@ func makeManifestJob( return } } else { - packageSpecs = mockDepsolve(manifest.GetPackageSetChains()) + packageSpecs, repoConfigs = mockDepsolve(manifest.GetPackageSetChains(), repos) } + _ = repoConfigs var containerSpecs map[string][]container.Spec if content["containers"] { @@ -274,7 +276,7 @@ func makeManifestJob( commitSpecs = mockResolveCommits(manifest.GetOSTreeSourceSpecs()) } - mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs) + mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs) if err != nil { return fmt.Errorf("[%s] manifest serialization failed: %s", filename, err.Error()) } @@ -378,21 +380,24 @@ func mockResolveCommits(commitSources map[string][]ostree.SourceSpec) map[string return commits } -func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) { +func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) for name, pkgSet := range packageSets { - res, err := solver.Depsolve(pkgSet) + packages, repos, err := solver.Depsolve(pkgSet) if err != nil { - return nil, err + return nil, nil, err } - depsolvedSets[name] = res + depsolvedSets[name] = packages + repoSets[name] = repos } - return depsolvedSets, nil + return depsolvedSets, repoSets, nil } -func mockDepsolve(packageSets map[string][]rpmmd.PackageSet) map[string][]rpmmd.PackageSpec { +func mockDepsolve(packageSets map[string][]rpmmd.PackageSet, repos []rpmmd.RepoConfig) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig) { depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) for name, pkgSetChain := range packageSets { specSet := make([]rpmmd.PackageSpec, 0) for _, pkgSet := range pkgSetChain { @@ -411,8 +416,10 @@ func mockDepsolve(packageSets map[string][]rpmmd.PackageSet) map[string][]rpmmd. } } depsolvedSets[name] = specSet + repoSets[name] = repos } - return depsolvedSets + + return depsolvedSets, repoSets } func save(ms manifest.OSBuildManifest, pkgs map[string][]rpmmd.PackageSpec, containers map[string][]container.Spec, commits map[string][]ostree.CommitSpec, cr buildRequest, path, filename string, metadata bool) error { diff --git a/cmd/osbuild-playground/playground.go b/cmd/osbuild-playground/playground.go index 44a6b28c39..ba6dab04ed 100644 --- a/cmd/osbuild-playground/playground.go +++ b/cmd/osbuild-playground/playground.go @@ -36,7 +36,7 @@ func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos packageSpecs := make(map[string][]rpmmd.PackageSpec) for name, chain := range manifest.GetPackageSetChains() { - packages, err := solver.Depsolve(chain) + packages, _, err := solver.Depsolve(chain) if err != nil { panic(fmt.Sprintf("failed to depsolve for pipeline %s: %s\n", name, err.Error())) } @@ -48,7 +48,7 @@ func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos fmt.Fprintf(os.Stderr, "could not clean dnf cache: %s", err.Error()) } - bytes, err := manifest.Serialize(packageSpecs, nil, nil) + bytes, err := manifest.Serialize(packageSpecs, nil, nil, nil) if err != nil { panic("failed to serialize manifest: " + err.Error()) } diff --git a/internal/mocks/rpmrepo/rpmrepo.go b/internal/mocks/rpmrepo/rpmrepo.go index c51ae84919..9fdc5e679d 100644 --- a/internal/mocks/rpmrepo/rpmrepo.go +++ b/internal/mocks/rpmrepo/rpmrepo.go @@ -1,7 +1,7 @@ package rpmrepo import ( - "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" @@ -31,18 +31,32 @@ func (trs *testRepoServer) Close() { trs.Server.Close() } -// WriteConfig writes the repository config to the file defined by the given -// path. Assumes the location already exists. +// WriteConfig writes the repository config to the specified path in .repo +// format. Assumes the location already exists. func (trs *testRepoServer) WriteConfig(path string) { - fp, err := os.Create(path) - if err != nil { - panic(err) + cfgtmpl := `[%[1]s] +name=%[1]s +baseurl=%[2]s +gpgcheck=%[3]s +sslverify=%[4]s +` + + checkGPG := "0" + if trs.RepoConfig.CheckGPG != nil && *trs.RepoConfig.CheckGPG { + checkGPG = "1" } - data, err := json.Marshal(trs.RepoConfig) + sslverify := "1" + if trs.RepoConfig.IgnoreSSL != nil && *trs.RepoConfig.IgnoreSSL { + sslverify = "0" + } + + config := fmt.Sprintf(cfgtmpl, trs.RepoConfig.Name, trs.RepoConfig.BaseURLs[0], checkGPG, sslverify) + + fp, err := os.Create(path) if err != nil { panic(err) } - if _, err := fp.Write(data); err != nil { + if _, err := fp.Write([]byte(config)); err != nil { panic(err) } } diff --git a/pkg/distro/distro_test.go b/pkg/distro/distro_test.go index 772685d6e7..631810f944 100644 --- a/pkg/distro/distro_test.go +++ b/pkg/distro/distro_test.go @@ -150,8 +150,10 @@ func TestImageTypePipelineNames(t *testing.T) { } packageSets := make(map[string][]rpmmd.PackageSpec, len(allPipelines)) + repoSets := make(map[string][]rpmmd.RepoConfig, len(allPipelines)) for _, plName := range allPipelines { packageSets[plName] = minimalPackageSet + repoSets[plName] = repos } m, _, err := imageType.Manifest(&bp, options, repos, seed) @@ -172,7 +174,7 @@ func TestImageTypePipelineNames(t *testing.T) { } commits[name] = commitSpecs } - mf, err := m.Serialize(packageSets, containers, commits) + mf, err := m.Serialize(packageSets, containers, commits, repoSets) assert.NoError(err) pm := new(manifest) err = json.Unmarshal(mf, pm) diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index 23b4c060f1..705a8ab955 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/osbuild/images/internal/common" "github.com/osbuild/images/pkg/rhsm" "github.com/osbuild/images/pkg/rpmmd" ) @@ -139,6 +140,8 @@ type Solver struct { // for each distribution. distro string + rootDir string + subscriptions *rhsm.Subscriptions } @@ -148,6 +151,13 @@ func NewSolver(modulePlatformID, releaseVer, arch, distro, cacheDir string) *Sol return s.NewWithConfig(modulePlatformID, releaseVer, arch, distro) } +// SetRootDir sets a path from which repository configurations, gpg keys, and +// vars are loaded during depsolve, instead of (or in addition to) the +// repositories and keys included in each depsolve request. +func (s *Solver) SetRootDir(path string) { + s.rootDir = path +} + // GetCacheDir returns a distro specific rpm cache directory // It ensures that the distro name is below the root cache directory, and if there is // a problem it returns the root cache instead of an error. @@ -164,10 +174,10 @@ func (s *Solver) GetCacheDir() string { // their associated repositories. Each package set is depsolved as a separate // transactions in a chain. It returns a list of all packages (with solved // dependencies) that will be installed into the system. -func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, error) { - req, repoMap, err := s.makeDepsolveRequest(pkgSets) +func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, []rpmmd.RepoConfig, error) { + req, rhsmMap, err := s.makeDepsolveRequest(pkgSets) if err != nil { - return nil, err + return nil, nil, err } // get non-exclusive read lock @@ -176,22 +186,25 @@ func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, erro output, err := run(s.dnfJsonCmd, req) if err != nil { - return nil, err + return nil, nil, err } // touch repos to now now := time.Now().Local() - for _, r := range repoMap { + for _, r := range req.Arguments.Repos { // ignore errors _ = s.cache.touchRepo(r.Hash(), now) } s.cache.updateInfo() - var result packageSpecs - if err := json.Unmarshal(output, &result); err != nil { - return nil, err + var result depsolveResult + dec := json.NewDecoder(bytes.NewReader(output)) + dec.DisallowUnknownFields() + if err := dec.Decode(&result); err != nil { + return nil, nil, err } - return result.toRPMMD(repoMap), nil + packages, repos := result.toRPMMD(rhsmMap) + return packages, repos, nil } // FetchMetadata returns the list of all the available packages in repos and @@ -309,15 +322,15 @@ func (s *Solver) reposFromRPMMD(rpmRepos []rpmmd.RepoConfig) ([]repoConfig, erro } if rr.CheckGPG != nil { - dr.CheckGPG = *rr.CheckGPG + dr.GPGCheck = *rr.CheckGPG } if rr.CheckRepoGPG != nil { - dr.CheckRepoGPG = *rr.CheckRepoGPG + dr.RepoGPGCheck = *rr.CheckRepoGPG } if rr.IgnoreSSL != nil { - dr.IgnoreSSL = *rr.IgnoreSSL + dr.SSLVerify = common.ToPtr(!*rr.IgnoreSSL) } if rr.RHSM { @@ -347,9 +360,9 @@ type repoConfig struct { Metalink string `json:"metalink,omitempty"` MirrorList string `json:"mirrorlist,omitempty"` GPGKeys []string `json:"gpgkeys,omitempty"` - CheckGPG bool `json:"gpgcheck"` - CheckRepoGPG bool `json:"check_repogpg"` - IgnoreSSL bool `json:"ignoressl"` + GPGCheck bool `json:"gpgcheck"` + RepoGPGCheck bool `json:"repo_gpgcheck"` + SSLVerify *bool `json:"sslverify,omitempty"` SSLCACert string `json:"sslcacert,omitempty"` SSLClientKey string `json:"sslclientkey,omitempty"` SSLClientCert string `json:"sslclientcert,omitempty"` @@ -366,30 +379,31 @@ func (r *repoConfig) Hash() string { return r.repoHash } -// Helper function for creating a depsolve request payload. -// The request defines a sequence of transactions, each depsolving one of the -// elements of `pkgSets` in the order they appear. The `repoConfigs` are used -// as the base repositories for all transactions. The extra repository configs -// in `pkgsetsRepos` are used for each of the `pkgSets` with matching index. -// The length of `pkgsetsRepos` must match the length of `pkgSets` or be empty -// (nil or empty slice). +// Helper function for creating a depsolve request payload. The request defines +// a sequence of transactions, each depsolving one of the elements of `pkgSets` +// in the order they appear. The repositories are collected in the request +// arguments indexed by their ID, and each transaction lists the repositories +// it will use for depsolving. +// +// The second return value is a map of repository IDs that have RHSM enabled. +// The RHSM property is not part of the dnf repository configuration so it's +// returned separately for setting the value on each package that requires it. // // NOTE: Due to implementation limitations of DNF and dnf-json, each package set // in the chain must use all of the repositories used by its predecessor. // An error is returned if this requirement is not met. -func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[string]rpmmd.RepoConfig, error) { - +func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[string]bool, error) { // dedupe repository configurations but maintain order // the order in which repositories are added to the request affects the // order of the dependencies in the result repos := make([]rpmmd.RepoConfig, 0) - rpmRepoMap := make(map[string]rpmmd.RepoConfig) + rhsmMap := make(map[string]bool) for _, ps := range pkgSets { for _, repo := range ps.Repositories { id := repo.Hash() - if _, ok := rpmRepoMap[id]; !ok { - rpmRepoMap[id] = repo + if _, ok := rhsmMap[id]; !ok { + rhsmMap[id] = repo.RHSM repos = append(repos, repo) } } @@ -429,6 +443,7 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[ } args := arguments{ Repos: dnfRepoMap, + RootDir: s.rootDir, Transactions: transactions, } @@ -436,11 +451,12 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[ Command: "depsolve", ModulePlatformID: s.modulePlatformID, Arch: s.arch, + Releasever: s.releaseVer, CacheDir: s.GetCacheDir(), Arguments: args, } - return &req, rpmRepoMap, nil + return &req, rhsmMap, nil } // Helper function for creating a dump request payload @@ -453,6 +469,7 @@ func (s *Solver) makeDumpRequest(repos []rpmmd.RepoConfig) (*Request, error) { Command: "dump", ModulePlatformID: s.modulePlatformID, Arch: s.arch, + Releasever: s.releaseVer, CacheDir: s.GetCacheDir(), Arguments: arguments{ Repos: dnfRepos, @@ -472,6 +489,7 @@ func (s *Solver) makeSearchRequest(repos []rpmmd.RepoConfig, packages []string) ModulePlatformID: s.modulePlatformID, Arch: s.arch, CacheDir: s.GetCacheDir(), + Releasever: s.releaseVer, Arguments: arguments{ Repos: dnfRepos, Search: searchArgs{ @@ -482,9 +500,12 @@ func (s *Solver) makeSearchRequest(repos []rpmmd.RepoConfig, packages []string) return &req, nil } -// convert internal a list of PackageSpecs to the rpmmd equivalent and attach -// key and subscription information based on the repository configs. -func (pkgs packageSpecs) toRPMMD(repos map[string]rpmmd.RepoConfig) []rpmmd.PackageSpec { +// convert internal a list of PackageSpecs and map of repoConfig to the rpmmd +// equivalents and attach key and subscription information based on the +// repository configs. +func (result depsolveResult) toRPMMD(rhsm map[string]bool) ([]rpmmd.PackageSpec, []rpmmd.RepoConfig) { + pkgs := result.Packages + repos := result.Repos rpmDependencies := make([]rpmmd.PackageSpec, len(pkgs)) for i, dep := range pkgs { repo, ok := repos[dep.RepoID] @@ -499,22 +520,46 @@ func (pkgs packageSpecs) toRPMMD(repos map[string]rpmmd.RepoConfig) []rpmmd.Pack rpmDependencies[i].Arch = dep.Arch rpmDependencies[i].RemoteLocation = dep.RemoteLocation rpmDependencies[i].Checksum = dep.Checksum - if repo.CheckGPG != nil { - rpmDependencies[i].CheckGPG = *repo.CheckGPG - } - if repo.IgnoreSSL != nil { - rpmDependencies[i].IgnoreSSL = *repo.IgnoreSSL + rpmDependencies[i].CheckGPG = repo.GPGCheck + if verify := repo.SSLVerify; verify != nil { + rpmDependencies[i].IgnoreSSL = !*verify } // The ssl secrets will also be set if rhsm is true, // which should take priority. - if repo.RHSM { + if rhsm[dep.RepoID] { rpmDependencies[i].Secrets = "org.osbuild.rhsm" } else if repo.SSLClientKey != "" { rpmDependencies[i].Secrets = "org.osbuild.mtls" } } - return rpmDependencies + + repoConfigs := make([]rpmmd.RepoConfig, 0, len(repos)) + for repoID := range repos { + repo := repos[repoID] + var ignoreSSL bool + if sslVerify := repo.SSLVerify; sslVerify != nil { + ignoreSSL = !*sslVerify + } + repoConfigs = append(repoConfigs, rpmmd.RepoConfig{ + Id: repo.ID, + Name: repo.Name, + BaseURLs: repo.BaseURLs, + Metalink: repo.Metalink, + MirrorList: repo.MirrorList, + GPGKeys: repo.GPGKeys, + CheckGPG: &repo.GPGCheck, + CheckRepoGPG: &repo.RepoGPGCheck, + IgnoreSSL: &ignoreSSL, + MetadataExpire: repo.MetadataExpire, + ModuleHotfixes: repo.ModuleHotfixes, + Enabled: common.ToPtr(true), + SSLCACert: repo.SSLCACert, + SSLClientKey: repo.SSLClientKey, + SSLClientCert: repo.SSLClientCert, + }) + } + return rpmDependencies, repoConfigs } // Request command and arguments for dnf-json @@ -525,6 +570,9 @@ type Request struct { // Platform ID, e.g., "platform:el8" ModulePlatformID string `json:"module_platform_id"` + // Distro Releasever, e.e., "8" + Releasever string `json:"releasever"` + // System architecture Arch string `json:"arch"` @@ -563,6 +611,10 @@ type arguments struct { // Depsolve package sets and repository mappings for this request Transactions []transactionArgs `json:"transactions"` + + // Load repository configurations, gpg keys, and vars from an os-root-like + // tree. + RootDir string `json:"root_dir"` } type searchArgs struct { @@ -591,6 +643,11 @@ type transactionArgs struct { type packageSpecs []PackageSpec +type depsolveResult struct { + Packages packageSpecs `json:"packages"` + Repos map[string]repoConfig `json:"repos"` +} + // Package specification type PackageSpec struct { Name string `json:"name"` diff --git a/pkg/dnfjson/dnfjson_test.go b/pkg/dnfjson/dnfjson_test.go index ff88a6e495..000dcaedf0 100644 --- a/pkg/dnfjson/dnfjson_test.go +++ b/pkg/dnfjson/dnfjson_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "flag" "fmt" + "os" + "path/filepath" "strings" "testing" @@ -11,6 +13,7 @@ import ( "github.com/osbuild/images/internal/mocks/rpmrepo" "github.com/osbuild/images/pkg/rpmmd" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var forceDNF = flag.Bool("force-dnf", false, "force dnf testing, making them fail instead of skip if dnf isn't installed") @@ -26,33 +29,92 @@ func TestDepsolver(t *testing.T) { s := rpmrepo.NewTestServer() defer s.Close() - assert := assert.New(t) + type testCase struct { + packages [][]string + repos []rpmmd.RepoConfig + rootDir string + err error + } tmpdir := t.TempDir() solver := NewSolver("platform:el9", "9", "x86_64", "rhel9.0", tmpdir) - { // single depsolve - pkgsets := []rpmmd.PackageSet{{Include: []string{"kernel", "vim-minimal", "tmux", "zsh"}, Repositories: []rpmmd.RepoConfig{s.RepoConfig}, InstallWeakDeps: true}} // everything you'll ever need + rootDir := t.TempDir() + reposDir := filepath.Join(rootDir, "etc", "yum.repos.d") + require.NoError(t, os.MkdirAll(reposDir, 0777)) + s.WriteConfig(filepath.Join(reposDir, "test.repo")) - deps, err := solver.Depsolve(pkgsets) - if err != nil { - t.Fatal(err) - } - exp := expectedResult(s.RepoConfig) - assert.Equal(deps, exp) + testCases := map[string]testCase{ + "flat": { + packages: [][]string{{"kernel", "vim-minimal", "tmux", "zsh"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, + err: nil, + }, + "chain": { + // chain depsolve of the same packages in order should produce the same result (at least in this case) + packages: [][]string{{"kernel"}, {"vim-minimal", "tmux", "zsh"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, + err: nil, + }, + "bad-flat": { + packages: [][]string{{"this-package-does-not-exist"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "bad-chain": { + packages: [][]string{{"kernel"}, {"this-package-does-not-exist"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "bad-chain-part-deux": { + packages: [][]string{{"this-package-does-not-exist"}, {"vim-minimal", "tmux", "zsh"}}, + repos: []rpmmd.RepoConfig{s.RepoConfig}, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "flat+dir": { + packages: [][]string{{"kernel", "vim-minimal", "tmux", "zsh"}}, + rootDir: rootDir, + err: nil, + }, + "chain+dir": { + packages: [][]string{{"kernel"}, {"vim-minimal", "tmux", "zsh"}}, + rootDir: rootDir, + err: nil, + }, + "bad-flat+dir": { + packages: [][]string{{"this-package-does-not-exist"}}, + rootDir: rootDir, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "bad-chain+dir": { + packages: [][]string{{"kernel"}, {"this-package-does-not-exist"}}, + rootDir: rootDir, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, + "bad-chain-part-deux+dir": { + packages: [][]string{{"this-package-does-not-exist"}, {"vim-minimal", "tmux", "zsh"}}, + rootDir: rootDir, + err: Error{Kind: "MarkingErrors", Reason: "Error occurred when marking packages for installation: Problems in request:\nmissing packages: this-package-does-not-exist"}, + }, } - { // chain depsolve of the same packages in order should produce the same result (at least in this case) - pkgsets := []rpmmd.PackageSet{ - {Include: []string{"kernel"}, Repositories: []rpmmd.RepoConfig{s.RepoConfig}, InstallWeakDeps: true}, - {Include: []string{"vim-minimal", "tmux", "zsh"}, Repositories: []rpmmd.RepoConfig{s.RepoConfig}}, - } - deps, err := solver.Depsolve(pkgsets) - if err != nil { - t.Fatal(err) - } - exp := expectedResult(s.RepoConfig) - assert.Equal(deps, exp) + for tcName := range testCases { + t.Run(tcName, func(t *testing.T) { + assert := assert.New(t) + tc := testCases[tcName] + pkgsets := make([]rpmmd.PackageSet, len(tc.packages)) + for idx := range tc.packages { + pkgsets[idx] = rpmmd.PackageSet{Include: tc.packages[idx], Repositories: tc.repos, InstallWeakDeps: true} + } + + solver.SetRootDir(tc.rootDir) + deps, _, err := solver.Depsolve(pkgsets) + assert.Equal(tc.err, err) + if err == nil { + exp := expectedResult(s.RepoConfig) + assert.Equal(deps, exp) + } + }) } } @@ -637,7 +699,7 @@ func TestErrorRepoInfo(t *testing.T) { solver := NewSolver("f38", "38", "x86_64", "fedora-38", "/tmp/cache") for idx, tc := range testCases { t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { - _, err := solver.Depsolve([]rpmmd.PackageSet{ + _, _, err := solver.Depsolve([]rpmmd.PackageSet{ { Include: []string{"osbuild"}, Exclude: nil, @@ -699,5 +761,5 @@ func TestRepoConfigMarshalAlsmostEmpty(t *testing.T) { repoCfg := &repoConfig{} js, _ := json.Marshal(repoCfg) // double check here that anything that uses pointers has "omitempty" set - assert.Equal(t, string(js), `{"id":"","gpgcheck":false,"check_repogpg":false,"ignoressl":false}`) + assert.Equal(t, string(js), `{"id":"","gpgcheck":false,"repo_gpgcheck":false}`) } diff --git a/pkg/image/bootc_disk_test.go b/pkg/image/bootc_disk_test.go index 7f8bdc1c68..81a0f3eca4 100644 --- a/pkg/image/bootc_disk_test.go +++ b/pkg/image/bootc_disk_test.go @@ -81,7 +81,7 @@ func makeBootcDiskImageOsbuildManifest(t *testing.T, opts *bootcDiskImageTestOpt "image": []container.Spec{{Source: "other-src", Digest: makeFakeDigest(t), ImageID: makeFakeDigest(t)}}, } - osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil) + osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil, nil) require.Nil(t, err) return osbuildManifest diff --git a/pkg/manifest/anaconda_installer.go b/pkg/manifest/anaconda_installer.go index 8342316f1f..204fdf571f 100644 --- a/pkg/manifest/anaconda_installer.go +++ b/pkg/manifest/anaconda_installer.go @@ -174,7 +174,7 @@ func (p *AnacondaInstaller) getPackageSpecs() []rpmmd.PackageSpec { return p.packageSpecs } -func (p *AnacondaInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) { +func (p *AnacondaInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } @@ -182,6 +182,7 @@ func (p *AnacondaInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []con if p.kernelName != "" { p.kernelVer = rpmmd.GetVerStrFromPackageSpecListPanic(p.packageSpecs, p.kernelName) } + p.repos = append(p.repos, rpmRepos...) } func (p *AnacondaInstaller) serializeEnd() { diff --git a/pkg/manifest/anaconda_installer_iso_tree.go b/pkg/manifest/anaconda_installer_iso_tree.go index ccd3f390bf..2d6f53cb86 100644 --- a/pkg/manifest/anaconda_installer_iso_tree.go +++ b/pkg/manifest/anaconda_installer_iso_tree.go @@ -156,7 +156,7 @@ func (p *AnacondaInstallerISOTree) getBuildPackages(_ Distro) []string { return packages } -func (p *AnacondaInstallerISOTree) serializeStart(_ []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) { +func (p *AnacondaInstallerISOTree) serializeStart(_ []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec, _ []rpmmd.RepoConfig) { if p.ostreeCommitSpec != nil || p.containerSpec != nil { panic("double call to serializeStart()") } diff --git a/pkg/manifest/anaconda_installer_iso_tree_test.go b/pkg/manifest/anaconda_installer_iso_tree_test.go index 6bebc43489..8d4055a12c 100644 --- a/pkg/manifest/anaconda_installer_iso_tree_test.go +++ b/pkg/manifest/anaconda_installer_iso_tree_test.go @@ -192,11 +192,11 @@ func TestAnacondaISOTreePayloadsBad(t *testing.T) { assert.PanicsWithValue( "pipeline supports at most one ostree commit", - func() { pipeline.serializeStart(nil, nil, make([]ostree.CommitSpec, 2)) }, + func() { pipeline.serializeStart(nil, nil, make([]ostree.CommitSpec, 2), nil) }, ) assert.PanicsWithValue( "pipeline supports at most one container", - func() { pipeline.serializeStart(nil, make([]container.Spec, 2), nil) }, + func() { pipeline.serializeStart(nil, make([]container.Spec, 2), nil, nil) }, ) } @@ -216,7 +216,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { t.Run("plain", func(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.OSPipeline = osPayload - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, payloadStages, @@ -229,7 +229,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.OSPipeline = osPayload pipeline.KSPath = testKsPath - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.kickstart"), @@ -242,7 +242,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { pipeline.OSPipeline = osPayload pipeline.KSPath = testKsPath pipeline.ISOLinux = true - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux", "org.osbuild.kickstart"), @@ -255,7 +255,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { pipeline.KSPath = testKsPath pipeline.ISOLinux = true pipeline.UnattendedKickstart = true - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux", "org.osbuild.kickstart"), @@ -270,7 +270,7 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { pipeline.ISOLinux = true pipeline.UnattendedKickstart = true pipeline.NoPasswd = []string{`%wheel`, `%sudo`} - pipeline.serializeStart(nil, nil, nil) + pipeline.serializeStart(nil, nil, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux", "org.osbuild.kickstart"), @@ -302,7 +302,7 @@ func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { t.Run("plain", func(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.KSPath = testKsPath - pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}) + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, payloadStages, @@ -314,7 +314,7 @@ func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.KSPath = testKsPath pipeline.ISOLinux = true - pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}) + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux"), variantStages)) @@ -325,7 +325,7 @@ func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { pipeline.KSPath = testKsPath pipeline.ISOLinux = true pipeline.UnattendedKickstart = true - pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}) + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux"), variantStages)) @@ -338,7 +338,7 @@ func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { pipeline.ISOLinux = true pipeline.UnattendedKickstart = true pipeline.NoPasswd = []string{`%wheel`, `%sudo`} - pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}) + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux"), variantStages)) @@ -369,7 +369,7 @@ func TestAnacondaISOTreeSerializeWithContainer(t *testing.T) { t.Run("kspath", func(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.KSPath = testKsPath - pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil) + pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, payloadStages, @@ -381,7 +381,7 @@ func TestAnacondaISOTreeSerializeWithContainer(t *testing.T) { pipeline := newTestAnacondaISOTree() pipeline.KSPath = testKsPath pipeline.ISOLinux = true - pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil) + pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil, nil) sp := pipeline.serialize() pipeline.serializeEnd() assert.NoError(t, checkISOTreeStages(sp.Stages, append(payloadStages, "org.osbuild.isolinux"), variantStages)) diff --git a/pkg/manifest/build.go b/pkg/manifest/build.go index ef82d47823..eca633d44a 100644 --- a/pkg/manifest/build.go +++ b/pkg/manifest/build.go @@ -99,11 +99,12 @@ func (p *BuildrootFromPackages) getPackageSpecs() []rpmmd.PackageSpec { return p.packageSpecs } -func (p *BuildrootFromPackages) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) { +func (p *BuildrootFromPackages) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } p.packageSpecs = packages + p.repos = append(p.repos, rpmRepos...) } func (p *BuildrootFromPackages) serializeEnd() { @@ -198,7 +199,7 @@ func (p *BuildrootFromContainer) getContainerSpecs() []container.Spec { return p.containerSpecs } -func (p *BuildrootFromContainer) serializeStart(_ []rpmmd.PackageSpec, containerSpecs []container.Spec, _ []ostree.CommitSpec) { +func (p *BuildrootFromContainer) serializeStart(_ []rpmmd.PackageSpec, containerSpecs []container.Spec, _ []ostree.CommitSpec, _ []rpmmd.RepoConfig) { if len(p.containerSpecs) > 0 { panic("double call to serializeStart()") } diff --git a/pkg/manifest/build_test.go b/pkg/manifest/build_test.go index ddfea70054..eb1da5680a 100644 --- a/pkg/manifest/build_test.go +++ b/pkg/manifest/build_test.go @@ -109,7 +109,7 @@ func TestNewBuildFromContainerSpecs(t *testing.T) { } // containerSpecs is "nil" until serializeStart populates it require.Nil(t, build.getContainerSpecs()) - build.serializeStart(nil, fakeContainerSpecs, nil) + build.serializeStart(nil, fakeContainerSpecs, nil, nil) assert.Equal(t, build.getContainerSpecs(), fakeContainerSpecs) osbuildPipeline := build.serialize() diff --git a/pkg/manifest/commit_server_tree.go b/pkg/manifest/commit_server_tree.go index b5233a3c0b..8f9c8669f0 100644 --- a/pkg/manifest/commit_server_tree.go +++ b/pkg/manifest/commit_server_tree.go @@ -80,11 +80,12 @@ func (p *OSTreeCommitServer) getPackageSpecs() []rpmmd.PackageSpec { return p.packageSpecs } -func (p *OSTreeCommitServer) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) { +func (p *OSTreeCommitServer) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } p.packageSpecs = packages + p.repos = append(p.repos, rpmRepos...) } func (p *OSTreeCommitServer) serializeEnd() { diff --git a/pkg/manifest/coreos_installer.go b/pkg/manifest/coreos_installer.go index a38f15aef0..26e208d0f7 100644 --- a/pkg/manifest/coreos_installer.go +++ b/pkg/manifest/coreos_installer.go @@ -136,7 +136,7 @@ func (p *CoreOSInstaller) getPackageSpecs() []rpmmd.PackageSpec { return p.packageSpecs } -func (p *CoreOSInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) { +func (p *CoreOSInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } @@ -144,6 +144,7 @@ func (p *CoreOSInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []conta if p.kernelName != "" { p.kernelVer = rpmmd.GetVerStrFromPackageSpecListPanic(p.packageSpecs, p.kernelName) } + p.repos = append(p.repos, rpmRepos...) } func (p *CoreOSInstaller) getInline() []string { diff --git a/pkg/manifest/empty.go b/pkg/manifest/empty.go index 8338e9d039..05c60cb4e1 100644 --- a/pkg/manifest/empty.go +++ b/pkg/manifest/empty.go @@ -22,6 +22,8 @@ type ContentTest struct { containerSpecs []container.Spec commitSpecs []ostree.CommitSpec + repos []rpmmd.RepoConfig + // serialization flag serializing bool } @@ -63,13 +65,14 @@ func (p *ContentTest) getOSTreeCommits() []ostree.CommitSpec { return p.commitSpecs } -func (p *ContentTest) serializeStart(pkgs []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) { +func (p *ContentTest) serializeStart(pkgs []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if p.serializing { panic("double call to serializeStart()") } p.packageSpecs = pkgs p.containerSpecs = containers p.commitSpecs = commits + p.repos = rpmRepos p.serializing = true } diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index 5a1f508a79..1ae10c5a0f 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -138,14 +138,14 @@ func (m Manifest) GetOSTreeSourceSpecs() map[string][]ostree.SourceSpec { return ostreeSpecs } -func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec) (OSBuildManifest, error) { +func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, rpmRepos map[string][]rpmmd.RepoConfig) (OSBuildManifest, error) { pipelines := make([]osbuild.Pipeline, 0) packages := make([]rpmmd.PackageSpec, 0) commits := make([]ostree.CommitSpec, 0) inline := make([]string, 0) containers := make([]container.Spec, 0) for _, pipeline := range m.pipelines { - pipeline.serializeStart(packageSets[pipeline.Name()], containerSpecs[pipeline.Name()], ostreeCommits[pipeline.Name()]) + pipeline.serializeStart(packageSets[pipeline.Name()], containerSpecs[pipeline.Name()], ostreeCommits[pipeline.Name()], rpmRepos[pipeline.Name()]) } for _, pipeline := range m.pipelines { commits = append(commits, pipeline.getOSTreeCommits()...) diff --git a/pkg/manifest/os.go b/pkg/manifest/os.go index b605abbb39..52f1549553 100644 --- a/pkg/manifest/os.go +++ b/pkg/manifest/os.go @@ -338,7 +338,7 @@ func (p *OS) getContainerSpecs() []container.Spec { return p.containerSpecs } -func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) { +func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec, rpmRepos []rpmmd.RepoConfig) { if len(p.packageSpecs) > 0 { panic("double call to serializeStart()") } @@ -355,6 +355,8 @@ func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container if p.KernelName != "" { p.kernelVer = rpmmd.GetVerStrFromPackageSpecListPanic(p.packageSpecs, p.KernelName) } + + p.repos = append(p.repos, rpmRepos...) } func (p *OS) serializeEnd() { diff --git a/pkg/manifest/os_test.go b/pkg/manifest/os_test.go index 87924a5e70..c6bd63e1cb 100644 --- a/pkg/manifest/os_test.go +++ b/pkg/manifest/os_test.go @@ -30,7 +30,7 @@ func NewTestOS() *OS { packages := []rpmmd.PackageSpec{ {Name: "pkg1", Checksum: "sha1:c02524e2bd19490f2a7167958f792262754c5f46"}, } - os.serializeStart(packages, nil, nil) + os.serializeStart(packages, nil, nil, repos) return os } diff --git a/pkg/manifest/ostree_deployment.go b/pkg/manifest/ostree_deployment.go index 40dbef5efa..a7518aa97a 100644 --- a/pkg/manifest/ostree_deployment.go +++ b/pkg/manifest/ostree_deployment.go @@ -161,7 +161,7 @@ func (p *OSTreeDeployment) getContainerSources() []container.SourceSpec { } } -func (p *OSTreeDeployment) serializeStart(packages []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) { +func (p *OSTreeDeployment) serializeStart(_ []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec, _ []rpmmd.RepoConfig) { if p.ostreeSpec != nil || p.containerSpec != nil { panic("double call to serializeStart()") } diff --git a/pkg/manifest/pipeline.go b/pkg/manifest/pipeline.go index ed1d67e79b..a9325693e6 100644 --- a/pkg/manifest/pipeline.go +++ b/pkg/manifest/pipeline.go @@ -53,7 +53,7 @@ type Pipeline interface { // its full Spec. See the ostree package for more details. getOSTreeCommitSources() []ostree.SourceSpec - serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec) + serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec, []rpmmd.RepoConfig) serializeEnd() serialize() osbuild.Pipeline @@ -166,7 +166,7 @@ func NewBase(name string, build Build) Base { // serializeStart must be called exactly once before each call // to serialize(). -func (p Base) serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec) { +func (p Base) serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec, []rpmmd.RepoConfig) { } // serializeEnd must be called exactly once after each call to diff --git a/pkg/manifest/raw_bootc.go b/pkg/manifest/raw_bootc.go index dfe4c2da75..1f9c93d823 100644 --- a/pkg/manifest/raw_bootc.go +++ b/pkg/manifest/raw_bootc.go @@ -66,7 +66,7 @@ func (p *RawBootcImage) getContainerSpecs() []container.Spec { return p.containerSpecs } -func (p *RawBootcImage) serializeStart(_ []rpmmd.PackageSpec, containerSpecs []container.Spec, _ []ostree.CommitSpec) { +func (p *RawBootcImage) serializeStart(_ []rpmmd.PackageSpec, containerSpecs []container.Spec, _ []ostree.CommitSpec, _ []rpmmd.RepoConfig) { if len(p.containerSpecs) > 0 { panic("double call to serializeStart()") } diff --git a/pkg/manifest/raw_bootc_export_test.go b/pkg/manifest/raw_bootc_export_test.go index 450bedd34d..2e87203c8a 100644 --- a/pkg/manifest/raw_bootc_export_test.go +++ b/pkg/manifest/raw_bootc_export_test.go @@ -15,6 +15,6 @@ func (rbc *RawBootcImage) Serialize() osbuild.Pipeline { return rbc.serialize() } -func (rbc *RawBootcImage) SerializeStart(a []rpmmd.PackageSpec, b []container.Spec, c []ostree.CommitSpec) { - rbc.serializeStart(a, b, c) +func (rbc *RawBootcImage) SerializeStart(a []rpmmd.PackageSpec, b []container.Spec, c []ostree.CommitSpec, r []rpmmd.RepoConfig) { + rbc.serializeStart(a, b, c, r) } diff --git a/pkg/manifest/raw_bootc_test.go b/pkg/manifest/raw_bootc_test.go index d144654ebb..94b94f8abb 100644 --- a/pkg/manifest/raw_bootc_test.go +++ b/pkg/manifest/raw_bootc_test.go @@ -51,7 +51,7 @@ func TestRawBootcImageSerialize(t *testing.T) { rawBootcPipeline.Users = []users.User{{Name: "root", Key: common.ToPtr("some-ssh-key")}} rawBootcPipeline.KernelOptionsAppend = []string{"karg1", "karg2"} - rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil) + rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil, nil) imagePipeline := rawBootcPipeline.Serialize() assert.Equal(t, "image", imagePipeline.Name) @@ -70,7 +70,7 @@ func TestRawBootcImageSerializeMountsValidated(t *testing.T) { rawBootcPipeline := manifest.NewRawBootcImage(build, nil, nil) // note that we create a partition table without /boot here rawBootcPipeline.PartitionTable = testdisk.MakeFakePartitionTable("/", "/missing-boot") - rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil) + rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil, nil) assert.PanicsWithError(t, `required mounts for bootupd stage [/boot /boot/efi] missing`, func() { rawBootcPipeline.Serialize() }) @@ -83,7 +83,7 @@ func TestRawBootcImageSerializeValidatesUsers(t *testing.T) { rawBootcPipeline := manifest.NewRawBootcImage(build, nil, nil) rawBootcPipeline.PartitionTable = testdisk.MakeFakePartitionTable("/", "/boot", "/boot/efi") - rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil) + rawBootcPipeline.SerializeStart(nil, []container.Spec{{Source: "foo"}}, nil, nil) for _, tc := range []struct { users []users.User diff --git a/test/scripts/generate-gitlab-ci b/test/scripts/generate-gitlab-ci index 979edd78cc..c5144e40b0 100755 --- a/test/scripts/generate-gitlab-ci +++ b/test/scripts/generate-gitlab-ci @@ -139,6 +139,7 @@ generate-manifests-{distro}-{arch}: RUNNER: aws/fedora-39-x86_64 INTERNAL_NETWORK: "true" script: + - sudo ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - go run ./cmd/gen-manifests --arches {arch} --distros {distro} --workers 10 --metadata=false --output ./manifests - for manifest in ./manifests/*; do diff --git a/test/scripts/imgtestlib.py b/test/scripts/imgtestlib.py index 0df01847b2..9ad6545347 100644 --- a/test/scripts/imgtestlib.py +++ b/test/scripts/imgtestlib.py @@ -432,12 +432,10 @@ def rng_seed_env(): def host_container_arch(): host_arch = os.uname().machine - match host_arch: - case "x86_64": - return "amd64" - case "aarch64": - return "arm64" - return host_arch + return { + "x86_64": "amd64", + "aarch64": "arm64" + }.get(host_arch, host_arch) def is_manifest_list(data):