diff --git a/cmd/osbuild-dnf-json-tests/main_test.go b/cmd/osbuild-dnf-json-tests/main_test.go index f2f34591b..46b1af941 100644 --- a/cmd/osbuild-dnf-json-tests/main_test.go +++ b/cmd/osbuild-dnf-json-tests/main_test.go @@ -80,12 +80,12 @@ func TestCrossArchDepsolve(t *testing.T) { imgType, err := arch.GetImageType(imgTypeStr) require.NoError(t, err) - buildPackages := imgType.BuildPackages() - _, _, err = rpm.Depsolve(buildPackages, []string{}, repos[archStr], distroStruct.ModulePlatformID(), archStr) + packages := imgType.PackageSets(blueprint.Blueprint{}) + + _, _, err = rpm.Depsolve(packages["build-packages"], repos[archStr], distroStruct.ModulePlatformID(), archStr) assert.NoError(t, err) - basePackagesInclude, basePackagesExclude := imgType.Packages(blueprint.Blueprint{}) - _, _, err = rpm.Depsolve(basePackagesInclude, basePackagesExclude, repos[archStr], distroStruct.ModulePlatformID(), archStr) + _, _, err = rpm.Depsolve(packages["packages"], repos[archStr], distroStruct.ModulePlatformID(), archStr) assert.NoError(t, err) }) } diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go index f361aff51..f380db06a 100644 --- a/cmd/osbuild-pipeline/main.go +++ b/cmd/osbuild-pipeline/main.go @@ -35,11 +35,6 @@ type composeRequest struct { Repositories []repository `json:"repositories"` } -type rpmMD struct { - BuildPackages []rpmmd.PackageSpec `json:"build-packages"` - Packages []rpmmd.PackageSpec `json:"packages"` -} - func main() { var rpmmdArg bool flag.BoolVar(&rpmmdArg, "rpmmd", false, "output rpmmd struct instead of pipeline manifest") @@ -116,32 +111,27 @@ func main() { } } - packages, excludePkgs := imageType.Packages(composeRequest.Blueprint) + packageSets := imageType.PackageSets(composeRequest.Blueprint) home, err := os.UserHomeDir() if err != nil { panic("os.UserHomeDir(): " + err.Error()) } - rpmmd := rpmmd.NewRPMMD(path.Join(home, ".cache/osbuild-composer/rpmmd"), "/usr/libexec/osbuild-composer/dnf-json") - packageSpecs, checksums, err := rpmmd.Depsolve(packages, excludePkgs, repos, d.ModulePlatformID(), arch.Name()) - if err != nil { - panic("Could not depsolve: " + err.Error()) - } + rpm_md := rpmmd.NewRPMMD(path.Join(home, ".cache/osbuild-composer/rpmmd"), "/usr/libexec/osbuild-composer/dnf-json") - buildPkgs := imageType.BuildPackages() - buildPackageSpecs, _, err := rpmmd.Depsolve(buildPkgs, nil, repos, d.ModulePlatformID(), arch.Name()) - if err != nil { - panic("Could not depsolve build packages: " + err.Error()) + packageSpecSets := make(map[string][]rpmmd.PackageSpec) + for name, packages := range packageSets { + packageSpecs, _, err := rpm_md.Depsolve(packages, repos, d.ModulePlatformID(), arch.Name()) + if err != nil { + panic("Could not depsolve: " + err.Error()) + } + packageSpecSets[name] = packageSpecs } var bytes []byte if rpmmdArg { - rpmMDInfo := rpmMD{ - BuildPackages: buildPackageSpecs, - Packages: packageSpecs, - } - bytes, err = json.Marshal(rpmMDInfo) + bytes, err = json.Marshal(packageSpecSets) if err != nil { panic(err) } @@ -151,8 +141,7 @@ func main() { Size: imageType.Size(0), }, repos, - packageSpecs, - buildPackageSpecs, + packageSpecSets, seedArg) if err != nil { panic(err.Error()) diff --git a/cmd/osbuild-store-dump/main.go b/cmd/osbuild-store-dump/main.go index a1c381fe0..290ffb350 100644 --- a/cmd/osbuild-store-dump/main.go +++ b/cmd/osbuild-store-dump/main.go @@ -16,17 +16,17 @@ import ( "github.com/osbuild/osbuild-composer/internal/target" ) -func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d distro.Distro, rpmmd rpmmd.RPMMD, repos []rpmmd.RepoConfig) distro.Manifest { - packages, excludePackages := t.Packages(bp) - pkgs, _, err := rpmmd.Depsolve(packages, excludePackages, repos, d.ModulePlatformID(), a.Name()) - if err != nil { - panic(err) +func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d distro.Distro, rpm_md rpmmd.RPMMD, repos []rpmmd.RepoConfig) distro.Manifest { + packageSets := t.PackageSets(bp) + pkgSpecSets := make(map[string][]rpmmd.PackageSpec) + for name, packages := range packageSets { + pkgs, _, err := rpm_md.Depsolve(packages, repos, d.ModulePlatformID(), a.Name()) + if err != nil { + panic(err) + } + pkgSpecSets[name] = pkgs } - buildPkgs, _, err := rpmmd.Depsolve(t.BuildPackages(), nil, repos, d.ModulePlatformID(), a.Name()) - if err != nil { - panic(err) - } - manifest, err := t.Manifest(bp.Customizations, distro.ImageOptions{}, repos, pkgs, buildPkgs, 0) + manifest, err := t.Manifest(bp.Customizations, distro.ImageOptions{}, repos, pkgSpecSets, 0) if err != nil { panic(err) } diff --git a/internal/cloudapi/server.go b/internal/cloudapi/server.go index 77b42b836..d931cddd6 100644 --- a/internal/cloudapi/server.go +++ b/internal/cloudapi/server.go @@ -132,17 +132,15 @@ func (server *Server) Compose(w http.ResponseWriter, r *http.Request) { } } - packageSpecs, excludePackageSpecs := imageType.Packages(bp) - packages, _, err := server.rpmMetadata.Depsolve(packageSpecs, excludePackageSpecs, repositories, distribution.ModulePlatformID(), arch.Name()) - if err != nil { - http.Error(w, fmt.Sprintf("Failed to depsolve base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err), http.StatusInternalServerError) - return - } - buildPackageSpecs := imageType.BuildPackages() - buildPackages, _, err := server.rpmMetadata.Depsolve(buildPackageSpecs, nil, repositories, distribution.ModulePlatformID(), arch.Name()) - if err != nil { - http.Error(w, fmt.Sprintf("Failed to depsolve build packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err), http.StatusInternalServerError) - return + packageSets := imageType.PackageSets(bp) + pkgSpecSets := make(map[string][]rpmmd.PackageSpec) + for name, packages := range packageSets { + pkgs, _, err := server.rpmMetadata.Depsolve(packages, repositories, distribution.ModulePlatformID(), arch.Name()) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to depsolve base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err), http.StatusInternalServerError) + return + } + pkgSpecSets[name] = pkgs } imageOptions := distro.ImageOptions{Size: imageType.Size(0)} @@ -156,7 +154,7 @@ func (server *Server) Compose(w http.ResponseWriter, r *http.Request) { } } - manifest, err := imageType.Manifest(nil, imageOptions, repositories, packages, buildPackages, manifestSeed) + manifest, err := imageType.Manifest(nil, imageOptions, repositories, pkgSpecSets, manifestSeed) if err != nil { http.Error(w, fmt.Sprintf("Failed to get manifest for for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err), http.StatusBadRequest) return diff --git a/internal/distro/distro.go b/internal/distro/distro.go index 3946ac97f..3db11d5e5 100644 --- a/internal/distro/distro.go +++ b/internal/distro/distro.go @@ -72,17 +72,16 @@ type ImageType interface { // is 0 the default value for the format will be returned. Size(size uint64) uint64 - // Returns the default packages to include and exclude when making the image - // type. - Packages(bp blueprint.Blueprint) ([]string, []string) - - // Returns the build packages for the output type. - BuildPackages() []string + // Returns the sets of packages to include and exclude when building the image. + // Indexed by a string label. How each set is labeled and used depends on the + // image type. + PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet // Returns an osbuild manifest, containing the sources and pipeline necessary // to build an image, given output format with all packages and customizations - // specified in the given blueprint. - Manifest(b *blueprint.Customizations, options ImageOptions, repos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, seed int64) (Manifest, error) + // specified in the given blueprint. The packageSpecSets must be labelled in + // the same way as the originating PackageSets. + Manifest(b *blueprint.Customizations, options ImageOptions, repos []rpmmd.RepoConfig, packageSpecSets map[string][]rpmmd.PackageSpec, seed int64) (Manifest, error) } // The ImageOptions specify options for a specific image build diff --git a/internal/distro/distro_test_common/distro_test_common.go b/internal/distro/distro_test_common/distro_test_common.go index 5dc932839..4121cc27b 100644 --- a/internal/distro/distro_test_common/distro_test_common.go +++ b/internal/distro/distro_test_common/distro_test_common.go @@ -38,14 +38,10 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, distr Repositories []repository `json:"repositories"` Blueprint *blueprint.Blueprint `json:"blueprint"` } - type rpmMD struct { - BuildPackages []rpmmd.PackageSpec `json:"build-packages"` - Packages []rpmmd.PackageSpec `json:"packages"` - } var tt struct { - ComposeRequest *composeRequest `json:"compose-request"` - RpmMD *rpmMD `json:"rpmmd"` - Manifest distro.Manifest `json:"manifest,omitempty"` + ComposeRequest *composeRequest `json:"compose-request"` + PackageSets map[string][]rpmmd.PackageSpec `json:"rpmmd"` + Manifest distro.Manifest `json:"manifest,omitempty"` } file, err := ioutil.ReadFile(fileName) assert.NoErrorf(err, "Could not read test-case '%s': %v", fileName, err) @@ -93,8 +89,7 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, distr }, }, repos, - tt.RpmMD.Packages, - tt.RpmMD.BuildPackages, + tt.PackageSets, RandomTestSeed) if (err == nil && tt.Manifest == nil) || (err != nil && tt.Manifest != nil) { @@ -116,7 +111,8 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, distr } func isOSTree(imgType distro.ImageType) bool { - for _, pkg := range imgType.BuildPackages() { + packageSets := imgType.PackageSets(blueprint.Blueprint{}) + for _, pkg := range packageSets["build-packages"].Include { if pkg == "rpm-ostree" { return true } @@ -128,7 +124,7 @@ var knownKernels = []string{"kernel", "kernel-debug", "kernel-rt"} // Returns the number of known kernels in the package list func kernelCount(imgType distro.ImageType) int { - pkgs, _ := imgType.Packages(blueprint.Blueprint{}) + pkgs := imgType.PackageSets(blueprint.Blueprint{})["packages"].Include n := 0 for _, pkg := range pkgs { for _, kernel := range knownKernels { diff --git a/internal/distro/fedora32/distro.go b/internal/distro/fedora32/distro.go index 69618b029..e3d0dee08 100644 --- a/internal/distro/fedora32/distro.go +++ b/internal/distro/fedora32/distro.go @@ -187,20 +187,32 @@ func (t *imageType) BuildPackages() []string { return packages } +func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet { + includePackages, excludePackages := t.Packages(bp) + return map[string]rpmmd.PackageSet{ + "packages": { + Include: includePackages, + Exclude: excludePackages, + }, + "build-packages": { + Include: t.BuildPackages(), + }, + } +} + func (t *imageType) Manifest(c *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, - packageSpecs, - buildPackageSpecs []rpmmd.PackageSpec, + packageSpecSets map[string][]rpmmd.PackageSpec, seed int64) (distro.Manifest, error) { - pipeline, err := t.pipeline(c, options, repos, packageSpecs, buildPackageSpecs) + pipeline, err := t.pipeline(c, options, repos, packageSpecSets["packages"], packageSpecSets["build-packages"]) if err != nil { return distro.Manifest{}, err } return json.Marshal( osbuild.Manifest{ - Sources: *sources(append(packageSpecs, buildPackageSpecs...)), + Sources: *sources(append(packageSpecSets["packages"], packageSpecSets["build-packages"]...)), Pipeline: *pipeline, }, ) diff --git a/internal/distro/fedora32/distro_test.go b/internal/distro/fedora32/distro_test.go index 9052747c1..36e4aef0c 100644 --- a/internal/distro/fedora32/distro_test.go +++ b/internal/distro/fedora32/distro_test.go @@ -121,13 +121,15 @@ func TestImageType_BuildPackages(t *testing.T) { t.Errorf("d.GetArch(%v) returned err = %v; expected nil", archLabel, err) continue } + buildPkgs := itStruct.PackageSets(blueprint.Blueprint{})["build-packages"] + assert.NotNil(t, buildPkgs) if itLabel == "fedora-iot-commit" { // For now we only include rpm-ostree when building fedora-iot-commit image types, this we may want // to reconsider. The only reason to specia-case it is that it might pull in a lot of dependencies // for a niche usecase. - assert.ElementsMatch(t, append(buildPackages[archLabel], "rpm-ostree"), itStruct.BuildPackages()) + assert.ElementsMatch(t, append(buildPackages[archLabel], "rpm-ostree"), buildPkgs.Include) } else { - assert.ElementsMatch(t, buildPackages[archLabel], itStruct.BuildPackages()) + assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs.Include) } } } @@ -280,15 +282,16 @@ func TestImageType_BasePackages(t *testing.T) { for _, pkgMap := range pkgMaps { imgType, err := arch.GetImageType(pkgMap.name) assert.NoError(t, err) - basePackages, excludedPackages := imgType.Packages(blueprint.Blueprint{}) + packages := imgType.PackageSets(blueprint.Blueprint{})["packages"] + assert.NotNil(t, packages) assert.Equalf( t, append(pkgMap.basePackages, pkgMap.bootloaderPackages...), - basePackages, + packages.Include, "image type: %s", pkgMap.name, ) - assert.Equalf(t, pkgMap.excludedPackages, excludedPackages, "image type: %s", pkgMap.name) + assert.Equalf(t, pkgMap.excludedPackages, packages.Exclude, "image type: %s", pkgMap.name) } } @@ -314,7 +317,7 @@ func TestDistro_ManifestError(t *testing.T) { arch, _ := f32distro.GetArch(archName) for _, imgTypeName := range arch.ListImageTypes() { imgType, _ := arch.GetImageType(imgTypeName) - _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, nil, 0) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, 0) if imgTypeName == "fedora-iot-commit" { assert.EqualError(t, err, "kernel boot parameter customizations are not supported for ostree types") } else { diff --git a/internal/distro/fedora33/distro.go b/internal/distro/fedora33/distro.go index a1867daaa..ca85bbf20 100644 --- a/internal/distro/fedora33/distro.go +++ b/internal/distro/fedora33/distro.go @@ -187,20 +187,32 @@ func (t *imageType) BuildPackages() []string { return packages } +func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet { + includePackages, excludePackages := t.Packages(bp) + return map[string]rpmmd.PackageSet{ + "packages": { + Include: includePackages, + Exclude: excludePackages, + }, + "build-packages": { + Include: t.BuildPackages(), + }, + } +} + func (t *imageType) Manifest(c *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, - packageSpecs, - buildPackageSpecs []rpmmd.PackageSpec, + packageSpecSets map[string][]rpmmd.PackageSpec, seed int64) (distro.Manifest, error) { - pipeline, err := t.pipeline(c, options, repos, packageSpecs, buildPackageSpecs) + pipeline, err := t.pipeline(c, options, repos, packageSpecSets["packages"], packageSpecSets["build-packages"]) if err != nil { return distro.Manifest{}, err } return json.Marshal( osbuild.Manifest{ - Sources: *sources(append(packageSpecs, buildPackageSpecs...)), + Sources: *sources(append(packageSpecSets["packages"], packageSpecSets["build-packages"]...)), Pipeline: *pipeline, }, ) diff --git a/internal/distro/fedora33/distro_test.go b/internal/distro/fedora33/distro_test.go index 3405ff900..e9ae5e2ce 100644 --- a/internal/distro/fedora33/distro_test.go +++ b/internal/distro/fedora33/distro_test.go @@ -121,13 +121,15 @@ func TestImageType_BuildPackages(t *testing.T) { t.Errorf("d.GetArch(%v) returned err = %v; expected nil", archLabel, err) continue } + buildPkgs := itStruct.PackageSets(blueprint.Blueprint{})["build-packages"] + assert.NotNil(t, buildPkgs) if itLabel == "fedora-iot-commit" { // For now we only include rpm-ostree when building fedora-iot-commit image types, this we may want // to reconsider. The only reason to specia-case it is that it might pull in a lot of dependencies // for a niche usecase. - assert.ElementsMatch(t, append(buildPackages[archLabel], "rpm-ostree"), itStruct.BuildPackages()) + assert.ElementsMatch(t, append(buildPackages[archLabel], "rpm-ostree"), buildPkgs.Include) } else { - assert.ElementsMatch(t, buildPackages[archLabel], itStruct.BuildPackages()) + assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs.Include) } } } @@ -286,15 +288,16 @@ func TestImageType_BasePackages(t *testing.T) { for _, pkgMap := range pkgMaps { imgType, err := arch.GetImageType(pkgMap.name) assert.NoError(t, err) - basePackages, excludedPackages := imgType.Packages(blueprint.Blueprint{}) + packages := imgType.PackageSets(blueprint.Blueprint{})["packages"] + assert.NotNil(t, packages) assert.Equalf( t, append(pkgMap.basePackages, pkgMap.bootloaderPackages...), - basePackages, + packages.Include, "image type: %s", pkgMap.name, ) - assert.Equalf(t, pkgMap.excludedPackages, excludedPackages, "image type: %s", pkgMap.name) + assert.Equalf(t, pkgMap.excludedPackages, packages.Exclude, "image type: %s", pkgMap.name) } } @@ -320,7 +323,7 @@ func TestDistro_ManifestError(t *testing.T) { arch, _ := f33distro.GetArch(archName) for _, imgTypeName := range arch.ListImageTypes() { imgType, _ := arch.GetImageType(imgTypeName) - _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, nil, 0) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, 0) if imgTypeName == "fedora-iot-commit" { assert.EqualError(t, err, "kernel boot parameter customizations are not supported for ostree types") } else { diff --git a/internal/distro/fedoratest/distro.go b/internal/distro/fedoratest/distro.go index 293852e43..1a22101e6 100644 --- a/internal/distro/fedoratest/distro.go +++ b/internal/distro/fedoratest/distro.go @@ -95,11 +95,14 @@ func (t *imageType) BuildPackages() []string { return nil } +func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet { + return nil +} + func (t *imageType) Manifest(c *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, - packageSpecs, - buildPackageSpecs []rpmmd.PackageSpec, + packageSpecSets map[string][]rpmmd.PackageSpec, seed int64) (distro.Manifest, error) { return json.Marshal( diff --git a/internal/distro/rhel8/distro.go b/internal/distro/rhel8/distro.go index c1a154cd5..7c7e8d4af 100644 --- a/internal/distro/rhel8/distro.go +++ b/internal/distro/rhel8/distro.go @@ -188,20 +188,32 @@ func (t *imageType) BuildPackages() []string { return packages } +func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet { + includePackages, excludePackages := t.Packages(bp) + return map[string]rpmmd.PackageSet{ + "packages": { + Include: includePackages, + Exclude: excludePackages, + }, + "build-packages": { + Include: t.BuildPackages(), + }, + } +} + func (t *imageType) Manifest(c *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, - packageSpecs, - buildPackageSpecs []rpmmd.PackageSpec, + packageSpecSets map[string][]rpmmd.PackageSpec, seed int64) (distro.Manifest, error) { - pipeline, err := t.pipeline(c, options, repos, packageSpecs, buildPackageSpecs) + pipeline, err := t.pipeline(c, options, repos, packageSpecSets["packages"], packageSpecSets["build-packages"]) if err != nil { return distro.Manifest{}, err } return json.Marshal( osbuild.Manifest{ - Sources: *sources(append(packageSpecs, buildPackageSpecs...)), + Sources: *sources(append(packageSpecSets["packages"], packageSpecSets["build-packages"]...)), Pipeline: *pipeline, }, ) diff --git a/internal/distro/rhel8/distro_test.go b/internal/distro/rhel8/distro_test.go index 5221e31ea..6305d07ac 100644 --- a/internal/distro/rhel8/distro_test.go +++ b/internal/distro/rhel8/distro_test.go @@ -126,7 +126,9 @@ func TestImageType_BuildPackages(t *testing.T) { if assert.NoErrorf(t, err, "d.GetArch(%v) returned err = %v; expected nil", archLabel, err) { continue } - assert.ElementsMatch(t, buildPackages[archLabel], itStruct.BuildPackages()) + buildPkgs := itStruct.PackageSets(blueprint.Blueprint{})["build-packages"] + assert.NotNil(t, buildPkgs) + assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs.Include) } } } @@ -339,15 +341,16 @@ func TestImageType_BasePackages(t *testing.T) { for _, pkgMap := range pkgMaps { imgType, err := arch.GetImageType(pkgMap.name) assert.NoError(t, err) - basePackages, excludedPackages := imgType.Packages(blueprint.Blueprint{}) + packages := imgType.PackageSets(blueprint.Blueprint{})["packages"] + assert.NotNil(t, packages) assert.Equalf( t, append(pkgMap.basePackages, pkgMap.bootloaderPackages...), - basePackages, + packages.Include, "image type: %s", pkgMap.name, ) - assert.Equalf(t, pkgMap.excludedPackages, excludedPackages, "image type: %s", pkgMap.name) + assert.Equalf(t, pkgMap.excludedPackages, packages.Exclude, "image type: %s", pkgMap.name) } } @@ -372,7 +375,7 @@ func TestDistro_ManifestError(t *testing.T) { arch, _ := r8distro.GetArch(archName) for _, imgTypeName := range arch.ListImageTypes() { imgType, _ := arch.GetImageType(imgTypeName) - _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, nil, 0) + _, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, 0) if imgTypeName == "rhel-edge-commit" { assert.EqualError(t, err, "kernel boot parameter customizations are not supported for ostree types") } else { diff --git a/internal/distro/rhel84/distro.go b/internal/distro/rhel84/distro.go index 5ca497ca6..e0ab7e15a 100644 --- a/internal/distro/rhel84/distro.go +++ b/internal/distro/rhel84/distro.go @@ -207,22 +207,34 @@ func (t *imageType) BuildPackages() []string { return packages } +func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet { + includePackages, excludePackages := t.Packages(bp) + return map[string]rpmmd.PackageSet{ + "packages": { + Include: includePackages, + Exclude: excludePackages, + }, + "build-packages": { + Include: t.BuildPackages(), + }, + } +} + func (t *imageType) Manifest(c *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, - packageSpecs, - buildPackageSpecs []rpmmd.PackageSpec, + packageSpecSets map[string][]rpmmd.PackageSpec, seed int64) (distro.Manifest, error) { source := rand.NewSource(seed) rng := rand.New(source) - pipeline, err := t.pipeline(c, options, repos, packageSpecs, buildPackageSpecs, rng) + pipeline, err := t.pipeline(c, options, repos, packageSpecSets["packages"], packageSpecSets["build-packages"], rng) if err != nil { return distro.Manifest{}, err } return json.Marshal( osbuild.Manifest{ - Sources: *sources(append(packageSpecs, buildPackageSpecs...)), + Sources: *sources(append(packageSpecSets["packages"], packageSpecSets["build-packages"]...)), Pipeline: *pipeline, }, ) diff --git a/internal/distro/rhel84/distro_test.go b/internal/distro/rhel84/distro_test.go index 05188d9bb..741dff5a2 100644 --- a/internal/distro/rhel84/distro_test.go +++ b/internal/distro/rhel84/distro_test.go @@ -153,7 +153,9 @@ func TestImageType_BuildPackages(t *testing.T) { if assert.NoErrorf(t, err, "d.GetArch(%v) returned err = %v; expected nil", archLabel, err) { continue } - assert.ElementsMatch(t, buildPackages[archLabel], itStruct.BuildPackages()) + buildPkgs := itStruct.PackageSets(blueprint.Blueprint{})["build-packages"] + assert.NotNil(t, buildPkgs) + assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs.Include) } } }) @@ -393,7 +395,8 @@ func TestImageType_BasePackages(t *testing.T) { for _, pkgMap := range pkgMaps { imgType, err := arch.GetImageType(pkgMap.name) assert.NoError(t, err) - basePackages, excludedPackages := imgType.Packages(blueprint.Blueprint{}) + packages := imgType.PackageSets(blueprint.Blueprint{})["packages"] + assert.NotNil(t, packages) expectedPackages := append(pkgMap.basePackages, pkgMap.bootloaderPackages...) if dist.name == "rhel" { expectedPackages = append(expectedPackages, pkgMap.rhelOnlyBasePackages...) @@ -401,11 +404,11 @@ func TestImageType_BasePackages(t *testing.T) { assert.ElementsMatchf( t, expectedPackages, - basePackages, + packages.Include, "image type: %s", pkgMap.name, ) - assert.Equalf(t, pkgMap.excludedPackages, excludedPackages, "image type: %s", pkgMap.name) + assert.Equalf(t, pkgMap.excludedPackages, packages.Exclude, "image type: %s", pkgMap.name) } }) } @@ -442,7 +445,7 @@ func TestDistro_ManifestError(t *testing.T) { imgOpts := distro.ImageOptions{ Size: imgType.Size(0), } - _, err := imgType.Manifest(bp.Customizations, imgOpts, nil, nil, nil, 0) + _, err := imgType.Manifest(bp.Customizations, imgOpts, nil, nil, 0) if imgTypeName == "rhel-edge-commit" { assert.EqualError(t, err, "kernel boot parameter customizations are not supported for ostree types") } else { diff --git a/internal/distro/test_distro/distro.go b/internal/distro/test_distro/distro.go index 3221aa5b4..c6b9348c9 100644 --- a/internal/distro/test_distro/distro.go +++ b/internal/distro/test_distro/distro.go @@ -79,7 +79,11 @@ func (t *TestImageType) BuildPackages() []string { return nil } -func (t *TestImageType) Manifest(b *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, seed int64) (distro.Manifest, error) { +func (t *TestImageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet { + return nil +} + +func (t *TestImageType) Manifest(b *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSpecSets map[string][]rpmmd.PackageSpec, seed int64) (distro.Manifest, error) { return json.Marshal( osbuild.Manifest{ Sources: osbuild.Sources{}, diff --git a/internal/kojiapi/server.go b/internal/kojiapi/server.go index a7b2c5089..1c4a5f2b7 100644 --- a/internal/kojiapi/server.go +++ b/internal/kojiapi/server.go @@ -118,18 +118,18 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error { if err != nil { panic("Could not initialize empty blueprint.") } - packageSpecs, excludePackageSpecs := imageType.Packages(*bp) - packages, _, err := h.server.rpmMetadata.Depsolve(packageSpecs, excludePackageSpecs, repositories, d.ModulePlatformID(), arch.Name()) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to depsolve base base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err)) - } - buildPackageSpecs := imageType.BuildPackages() - buildPackages, _, err := h.server.rpmMetadata.Depsolve(buildPackageSpecs, nil, repositories, d.ModulePlatformID(), arch.Name()) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to depsolve build packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err)) + + packageSets := imageType.PackageSets(*bp) + packageSpecSets := make(map[string][]rpmmd.PackageSpec) + for name, packages := range packageSets { + packageSpecs, _, err := h.server.rpmMetadata.Depsolve(packages, repositories, d.ModulePlatformID(), arch.Name()) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to depsolve base base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err)) + } + packageSpecSets[name] = packageSpecs } - manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, repositories, packages, buildPackages, manifestSeed) + manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, repositories, packageSpecSets, manifestSeed) if err != nil { return echo.NewHTTPError(http.StatusBadGateway, fmt.Sprintf("Failed to get manifest for for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err)) } diff --git a/internal/mocks/rpmmd/rpmmd_mock.go b/internal/mocks/rpmmd/rpmmd_mock.go index 49049f72c..dd42be020 100644 --- a/internal/mocks/rpmmd/rpmmd_mock.go +++ b/internal/mocks/rpmmd/rpmmd_mock.go @@ -36,6 +36,6 @@ func (r *rpmmdMock) FetchMetadata(repos []rpmmd.RepoConfig, modulePlatformID str return r.Fixture.fetchPackageList.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.fetchPackageList.err } -func (r *rpmmdMock) Depsolve(specs, excludeSpecs []string, repos []rpmmd.RepoConfig, modulePlatformID, arch string) ([]rpmmd.PackageSpec, map[string]string, error) { +func (r *rpmmdMock) Depsolve(packageSet rpmmd.PackageSet, repos []rpmmd.RepoConfig, modulePlatformID, arch string) ([]rpmmd.PackageSpec, map[string]string, error) { return r.Fixture.depsolve.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.depsolve.err } diff --git a/internal/rpmmd/repository.go b/internal/rpmmd/repository.go index a8a895c1c..b048fb480 100644 --- a/internal/rpmmd/repository.go +++ b/internal/rpmmd/repository.go @@ -96,6 +96,13 @@ func (pkg Package) ToPackageInfo() PackageInfo { } } +// The inputs to depsolve, a set of packages to include and a set of +// packages to exclude. +type PackageSet struct { + Include []string + Exclude []string +} + // TODO: the public API of this package should not be reused for serialization. type PackageSpec struct { Name string `json:"name"` @@ -159,7 +166,7 @@ type RPMMD interface { // Depsolve takes a list of required content (specs), explicitly unwanted content (excludeSpecs), list // or repositories, and platform ID for modularity. It returns a list of all packages (with solved // dependencies) that will be installed into the system. - Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID, arch string) ([]PackageSpec, map[string]string, error) + Depsolve(packageSet PackageSet, repos []RepoConfig, modulePlatformID, arch string) ([]PackageSpec, map[string]string, error) } type DNFError struct { @@ -377,7 +384,7 @@ func (r *rpmmdImpl) FetchMetadata(repos []RepoConfig, modulePlatformID string, a return reply.Packages, checksums, err } -func (r *rpmmdImpl) Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID, arch string) ([]PackageSpec, map[string]string, error) { +func (r *rpmmdImpl) Depsolve(packageSet PackageSet, repos []RepoConfig, modulePlatformID, arch string) ([]PackageSpec, map[string]string, error) { var dnfRepoConfigs []dnfRepoConfig for i, repo := range repos { @@ -395,7 +402,7 @@ func (r *rpmmdImpl) Depsolve(specs, excludeSpecs []string, repos []RepoConfig, m CacheDir string `json:"cachedir"` ModulePlatformID string `json:"module_platform_id"` Arch string `json:"arch"` - }{specs, excludeSpecs, dnfRepoConfigs, r.CacheDir, modulePlatformID, arch} + }{packageSet.Include, packageSet.Exclude, dnfRepoConfigs, r.CacheDir, modulePlatformID, arch} var reply struct { Checksums map[string]string `json:"checksums"` Dependencies []dnfPackageSpec `json:"dependencies"` @@ -479,6 +486,6 @@ func (packages PackageList) ToPackageInfos() []PackageInfo { } func (pkg *PackageInfo) FillDependencies(rpmmd RPMMD, repos []RepoConfig, modulePlatformID string, arch string) (err error) { - pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, nil, repos, modulePlatformID, arch) + pkg.Dependencies, _, err = rpmmd.Depsolve(PackageSet{Include: []string{pkg.Name}}, repos, modulePlatformID, arch) return } diff --git a/internal/store/fixtures.go b/internal/store/fixtures.go index f72e8a592..759d9f250 100644 --- a/internal/store/fixtures.go +++ b/internal/store/fixtures.go @@ -57,7 +57,7 @@ func FixtureBase() *Store { if err != nil { panic("invalid image type qcow2 for x86_64 @ fedoratest") } - manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, nil, 0) + manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, 0) if err != nil { panic("could not create manifest") } @@ -160,7 +160,7 @@ func FixtureFinished() *Store { if err != nil { panic("invalid image type qcow2 for x86_64 @ fedoratest") } - manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, nil, 0) + manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, 0) if err != nil { panic("could not create manifest") } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 9a5844149..b836e1f93 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -51,7 +51,7 @@ func (suite *storeTest) SetupSuite() { suite.myDistro = test_distro.New() suite.myArch, _ = suite.myDistro.GetArch("test_arch") suite.myImageType, _ = suite.myArch.GetImageType("test_type") - suite.myManifest, _ = suite.myImageType.Manifest(&suite.myCustomizations, suite.myImageOptions, suite.myRepoConfig, nil, suite.myPackageSpec, 0) + suite.myManifest, _ = suite.myImageType.Manifest(&suite.myCustomizations, suite.myImageOptions, suite.myRepoConfig, nil, 0) suite.mySourceConfig = SourceConfig{ Name: "testSourceConfig", } diff --git a/internal/weldr/api.go b/internal/weldr/api.go index 40b7a365f..829e1868a 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -1038,7 +1038,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt projects = projects[1:] names := strings.Split(projects, ",") - packages, _, err := api.rpmmd.Depsolve(names, nil, api.repos, api.distro.ModulePlatformID(), api.arch.Name()) + packages, _, err := api.rpmmd.Depsolve(rpmmd.PackageSet{Include: names}, api.repos, api.distro.ModulePlatformID(), api.arch.Name()) if err != nil { errors := responseError{ @@ -1233,7 +1233,7 @@ func (api *API) blueprintsDepsolveHandler(writer http.ResponseWriter, request *h continue } - dependencies, _, err := api.depsolveBlueprint(blueprint, nil) + dependencies, err := api.depsolveBlueprint(blueprint) if err != nil { blueprintsErrors = append(blueprintsErrors, responseError{ @@ -1322,7 +1322,7 @@ func (api *API) blueprintsFreezeHandler(writer http.ResponseWriter, request *htt } // Make a copy of the blueprint since we will be replacing the version globs blueprint := bp.DeepCopy() - dependencies, _, err := api.depsolveBlueprint(&blueprint, nil) + dependencies, err := api.depsolveBlueprint(&blueprint) if err != nil { rerr := responseError{ ID: "BlueprintsError", @@ -1829,6 +1829,19 @@ func ostreeResolveRef(location, ref string) (string, error) { return parent, nil } +func (api *API) depsolveBlueprintForImageType(bp *blueprint.Blueprint, imageType distro.ImageType) (map[string][]rpmmd.PackageSpec, error) { + packageSets := imageType.PackageSets(*bp) + packageSpecSets := make(map[string][]rpmmd.PackageSpec) + for name, packageSet := range packageSets { + packageSpecs, _, err := api.rpmmd.Depsolve(packageSet, api.allRepositories(), api.distro.ModulePlatformID(), api.arch.Name()) + if err != nil { + return nil, err + } + packageSpecSets[name] = packageSpecs + } + return packageSpecSets, nil +} + // Schedule new compose by first translating the appropriate blueprint into a pipeline and then // pushing it into the channel for waiting builds. func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { @@ -1959,7 +1972,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request cr.OSTree.Parent = parent } - packages, buildPackages, err := api.depsolveBlueprint(bp, imageType) + packageSets, err := api.depsolveBlueprintForImageType(bp, imageType) if err != nil { errors := responseError{ ID: "DepsolveError", @@ -1986,8 +1999,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request }, }, api.allRepositories(), - packages, - buildPackages, + packageSets, seed) if err != nil { errors := responseError{ @@ -2738,32 +2750,13 @@ func (api *API) allRepositories() []rpmmd.RepoConfig { return repos } -func (api *API) depsolveBlueprint(bp *blueprint.Blueprint, imageType distro.ImageType) ([]rpmmd.PackageSpec, []rpmmd.PackageSpec, error) { - repos := api.allRepositories() - - specs := bp.GetPackages() - excludeSpecs := []string{} - if imageType != nil { - // When the output type is known, include the base packages in the depsolve - // transaction. - specs, excludeSpecs = imageType.Packages(*bp) - } - - packages, _, err := api.rpmmd.Depsolve(specs, excludeSpecs, repos, api.distro.ModulePlatformID(), api.arch.Name()) +func (api *API) depsolveBlueprint(bp *blueprint.Blueprint) ([]rpmmd.PackageSpec, error) { + packages, _, err := api.rpmmd.Depsolve(rpmmd.PackageSet{Include: bp.GetPackages()}, api.allRepositories(), api.distro.ModulePlatformID(), api.arch.Name()) if err != nil { - return nil, nil, err + return nil, err } - buildPackages := []rpmmd.PackageSpec{} - if imageType != nil { - buildSpecs := imageType.BuildPackages() - buildPackages, _, err = api.rpmmd.Depsolve(buildSpecs, nil, repos, api.distro.ModulePlatformID(), api.arch.Name()) - if err != nil { - return nil, nil, err - } - } - - return packages, buildPackages, err + return packages, err } func (api *API) uploadsScheduleHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) { diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go index 053066ba9..535122659 100644 --- a/internal/weldr/api_test.go +++ b/internal/weldr/api_test.go @@ -578,7 +578,7 @@ func TestCompose(t *testing.T) { require.NoError(t, err) imgType, err := arch.GetImageType("qcow2") require.NoError(t, err) - manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, nil, 0) + manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, 0) require.NoError(t, err) expectedComposeLocal := &store.Compose{ Blueprint: &blueprint.Blueprint{ diff --git a/internal/worker/server_test.go b/internal/worker/server_test.go index 59df3b6e0..9bf04c7ec 100644 --- a/internal/worker/server_test.go +++ b/internal/worker/server_test.go @@ -83,7 +83,7 @@ func TestCreate(t *testing.T) { if err != nil { t.Fatalf("error getting image type from arch") } - manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0) + manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0) if err != nil { t.Fatalf("error creating osbuild manifest") } @@ -111,7 +111,7 @@ func TestCancel(t *testing.T) { if err != nil { t.Fatalf("error getting image type from arch") } - manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0) + manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0) if err != nil { t.Fatalf("error creating osbuild manifest") } @@ -152,7 +152,7 @@ func TestUpdate(t *testing.T) { if err != nil { t.Fatalf("error getting image type from arch") } - manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0) + manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0) if err != nil { t.Fatalf("error creating osbuild manifest") } @@ -179,7 +179,7 @@ func TestArgs(t *testing.T) { require.NoError(t, err) imageType, err := arch.GetImageType("qcow2") require.NoError(t, err) - manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0) + manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0) require.NoError(t, err) tempdir, err := ioutil.TempDir("", "worker-tests-") @@ -221,7 +221,7 @@ func TestUpload(t *testing.T) { if err != nil { t.Fatalf("error getting image type from arch") } - manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0) + manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0) if err != nil { t.Fatalf("error creating osbuild manifest") }