particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

distro: introduce PackageSets

Browse source 
This replaces Packages() and BuildPackages() by returning a map of
package sets, the semantics of which is up to the distro to define.

They are meant to be depsolved and the result returned back as a
map to Manifest(), with the same keys.

No functional change.

Signed-off-by: Tom Gundersen <teg@jklm.no>
This commit is contained in:
Tom Gundersen 2021-03-09 17:35:28 +00:00
parent 4805eeedf8
commit 9e2e009ac8
24 changed files with 210 additions and 161 deletions
Download patch file Download diff file Expand all files Collapse all files

8
cmd/osbuild-dnf-json-tests/main_test.go
View file

@ -80,12 +80,12 @@ func TestCrossArchDepsolve(t *testing.T) {
imgType, err := arch.GetImageType(imgTypeStr)
require.NoError(t, err)
buildPackages := imgType.BuildPackages()
_, _, err = rpm.Depsolve(buildPackages, []string{}, repos[archStr], distroStruct.ModulePlatformID(), archStr)
packages := imgType.PackageSets(blueprint.Blueprint{})
_, _, err = rpm.Depsolve(packages["build-packages"], repos[archStr], distroStruct.ModulePlatformID(), archStr)
assert.NoError(t, err)
basePackagesInclude, basePackagesExclude := imgType.Packages(blueprint.Blueprint{})
_, _, err = rpm.Depsolve(basePackagesInclude, basePackagesExclude, repos[archStr], distroStruct.ModulePlatformID(), archStr)
_, _, err = rpm.Depsolve(packages["packages"], repos[archStr], distroStruct.ModulePlatformID(), archStr)
assert.NoError(t, err)
})
}

33
cmd/osbuild-pipeline/main.go
View file

@ -35,11 +35,6 @@ type composeRequest struct {
Repositories []repository `json:"repositories"`
}
type rpmMD struct {
BuildPackages []rpmmd.PackageSpec `json:"build-packages"`
Packages []rpmmd.PackageSpec `json:"packages"`
}
func main() {
var rpmmdArg bool
flag.BoolVar(&rpmmdArg, "rpmmd", false, "output rpmmd struct instead of pipeline manifest")
@ -116,32 +111,27 @@ func main() {
}
}
packages, excludePkgs := imageType.Packages(composeRequest.Blueprint)
packageSets := imageType.PackageSets(composeRequest.Blueprint)
home, err := os.UserHomeDir()
if err != nil {
panic("os.UserHomeDir(): " + err.Error())
}
rpmmd := rpmmd.NewRPMMD(path.Join(home, ".cache/osbuild-composer/rpmmd"), "/usr/libexec/osbuild-composer/dnf-json")
packageSpecs, checksums, err := rpmmd.Depsolve(packages, excludePkgs, repos, d.ModulePlatformID(), arch.Name())
if err != nil {
panic("Could not depsolve: " + err.Error())
}
rpm_md := rpmmd.NewRPMMD(path.Join(home, ".cache/osbuild-composer/rpmmd"), "/usr/libexec/osbuild-composer/dnf-json")
buildPkgs := imageType.BuildPackages()
buildPackageSpecs, _, err := rpmmd.Depsolve(buildPkgs, nil, repos, d.ModulePlatformID(), arch.Name())
if err != nil {
panic("Could not depsolve build packages: " + err.Error())
packageSpecSets := make(map[string][]rpmmd.PackageSpec)
for name, packages := range packageSets {
packageSpecs, _, err := rpm_md.Depsolve(packages, repos, d.ModulePlatformID(), arch.Name())
if err != nil {
panic("Could not depsolve: " + err.Error())
}
packageSpecSets[name] = packageSpecs
}
var bytes []byte
if rpmmdArg {
rpmMDInfo := rpmMD{
BuildPackages: buildPackageSpecs,
Packages: packageSpecs,
}
bytes, err = json.Marshal(rpmMDInfo)
bytes, err = json.Marshal(packageSpecSets)
if err != nil {
panic(err)
}
@ -151,8 +141,7 @@ func main() {
Size: imageType.Size(0),
},
repos,
packageSpecs,
buildPackageSpecs,
packageSpecSets,
seedArg)
if err != nil {
panic(err.Error())

20
cmd/osbuild-store-dump/main.go
View file

@ -16,17 +16,17 @@ import (
"github.com/osbuild/osbuild-composer/internal/target"
)
func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d distro.Distro, rpmmd rpmmd.RPMMD, repos []rpmmd.RepoConfig) distro.Manifest {
packages, excludePackages := t.Packages(bp)
pkgs, _, err := rpmmd.Depsolve(packages, excludePackages, repos, d.ModulePlatformID(), a.Name())
if err != nil {
panic(err)
func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d distro.Distro, rpm_md rpmmd.RPMMD, repos []rpmmd.RepoConfig) distro.Manifest {
packageSets := t.PackageSets(bp)
pkgSpecSets := make(map[string][]rpmmd.PackageSpec)
for name, packages := range packageSets {
pkgs, _, err := rpm_md.Depsolve(packages, repos, d.ModulePlatformID(), a.Name())
if err != nil {
panic(err)
}
pkgSpecSets[name] = pkgs
}
buildPkgs, _, err := rpmmd.Depsolve(t.BuildPackages(), nil, repos, d.ModulePlatformID(), a.Name())
if err != nil {
panic(err)
}
manifest, err := t.Manifest(bp.Customizations, distro.ImageOptions{}, repos, pkgs, buildPkgs, 0)
manifest, err := t.Manifest(bp.Customizations, distro.ImageOptions{}, repos, pkgSpecSets, 0)
if err != nil {
panic(err)
}

22
internal/cloudapi/server.go
View file

@ -132,17 +132,15 @@ func (server *Server) Compose(w http.ResponseWriter, r *http.Request) {
}
}
packageSpecs, excludePackageSpecs := imageType.Packages(bp)
packages, _, err := server.rpmMetadata.Depsolve(packageSpecs, excludePackageSpecs, repositories, distribution.ModulePlatformID(), arch.Name())
if err != nil {
http.Error(w, fmt.Sprintf("Failed to depsolve base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err), http.StatusInternalServerError)
return
}
buildPackageSpecs := imageType.BuildPackages()
buildPackages, _, err := server.rpmMetadata.Depsolve(buildPackageSpecs, nil, repositories, distribution.ModulePlatformID(), arch.Name())
if err != nil {
http.Error(w, fmt.Sprintf("Failed to depsolve build packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err), http.StatusInternalServerError)
return
packageSets := imageType.PackageSets(bp)
pkgSpecSets := make(map[string][]rpmmd.PackageSpec)
for name, packages := range packageSets {
pkgs, _, err := server.rpmMetadata.Depsolve(packages, repositories, distribution.ModulePlatformID(), arch.Name())
if err != nil {
http.Error(w, fmt.Sprintf("Failed to depsolve base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err), http.StatusInternalServerError)
return
}
pkgSpecSets[name] = pkgs
}
imageOptions := distro.ImageOptions{Size: imageType.Size(0)}
@ -156,7 +154,7 @@ func (server *Server) Compose(w http.ResponseWriter, r *http.Request) {
}
}
manifest, err := imageType.Manifest(nil, imageOptions, repositories, packages, buildPackages, manifestSeed)
manifest, err := imageType.Manifest(nil, imageOptions, repositories, pkgSpecSets, manifestSeed)
if err != nil {
http.Error(w, fmt.Sprintf("Failed to get manifest for for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err), http.StatusBadRequest)
return

15
internal/distro/distro.go
View file

@ -72,17 +72,16 @@ type ImageType interface {
// is 0 the default value for the format will be returned.
Size(size uint64) uint64
// Returns the default packages to include and exclude when making the image
// type.
Packages(bp blueprint.Blueprint) ([]string, []string)
// Returns the build packages for the output type.
BuildPackages() []string
// Returns the sets of packages to include and exclude when building the image.
// Indexed by a string label. How each set is labeled and used depends on the
// image type.
PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet
// Returns an osbuild manifest, containing the sources and pipeline necessary
// to build an image, given output format with all packages and customizations
// specified in the given blueprint.
Manifest(b *blueprint.Customizations, options ImageOptions, repos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, seed int64) (Manifest, error)
// specified in the given blueprint. The packageSpecSets must be labelled in
// the same way as the originating PackageSets.
Manifest(b *blueprint.Customizations, options ImageOptions, repos []rpmmd.RepoConfig, packageSpecSets map[string][]rpmmd.PackageSpec, seed int64) (Manifest, error)
}
// The ImageOptions specify options for a specific image build

18
internal/distro/distro_test_common/distro_test_common.go
View file

@ -38,14 +38,10 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, distr
Repositories []repository `json:"repositories"`
Blueprint *blueprint.Blueprint `json:"blueprint"`
}
type rpmMD struct {
BuildPackages []rpmmd.PackageSpec `json:"build-packages"`
Packages []rpmmd.PackageSpec `json:"packages"`
}
var tt struct {
ComposeRequest *composeRequest `json:"compose-request"`
RpmMD *rpmMD `json:"rpmmd"`
Manifest distro.Manifest `json:"manifest,omitempty"`
ComposeRequest *composeRequest `json:"compose-request"`
PackageSets map[string][]rpmmd.PackageSpec `json:"rpmmd"`
Manifest distro.Manifest `json:"manifest,omitempty"`
}
file, err := ioutil.ReadFile(fileName)
assert.NoErrorf(err, "Could not read test-case '%s': %v", fileName, err)
@ -93,8 +89,7 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, distr
},
},
repos,
tt.RpmMD.Packages,
tt.RpmMD.BuildPackages,
tt.PackageSets,
RandomTestSeed)
if (err == nil && tt.Manifest == nil) || (err != nil && tt.Manifest != nil) {
@ -116,7 +111,8 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, distr
}
func isOSTree(imgType distro.ImageType) bool {
for _, pkg := range imgType.BuildPackages() {
packageSets := imgType.PackageSets(blueprint.Blueprint{})
for _, pkg := range packageSets["build-packages"].Include {
if pkg == "rpm-ostree" {
return true
}
@ -128,7 +124,7 @@ var knownKernels = []string{"kernel", "kernel-debug", "kernel-rt"}
// Returns the number of known kernels in the package list
func kernelCount(imgType distro.ImageType) int {
pkgs, _ := imgType.Packages(blueprint.Blueprint{})
pkgs := imgType.PackageSets(blueprint.Blueprint{})["packages"].Include
n := 0
for _, pkg := range pkgs {
for _, kernel := range knownKernels {

20
internal/distro/fedora32/distro.go
View file

@ -187,20 +187,32 @@ func (t *imageType) BuildPackages() []string {
return packages
}
func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet {
includePackages, excludePackages := t.Packages(bp)
return map[string]rpmmd.PackageSet{
"packages": {
Include: includePackages,
Exclude: excludePackages,
},
"build-packages": {
Include: t.BuildPackages(),
},
}
}
func (t *imageType) Manifest(c *blueprint.Customizations,
options distro.ImageOptions,
repos []rpmmd.RepoConfig,
packageSpecs,
buildPackageSpecs []rpmmd.PackageSpec,
packageSpecSets map[string][]rpmmd.PackageSpec,
seed int64) (distro.Manifest, error) {
pipeline, err := t.pipeline(c, options, repos, packageSpecs, buildPackageSpecs)
pipeline, err := t.pipeline(c, options, repos, packageSpecSets["packages"], packageSpecSets["build-packages"])
if err != nil {
return distro.Manifest{}, err
}
return json.Marshal(
osbuild.Manifest{
Sources: *sources(append(packageSpecs, buildPackageSpecs...)),
Sources: *sources(append(packageSpecSets["packages"], packageSpecSets["build-packages"]...)),
Pipeline: *pipeline,
},
)

15
internal/distro/fedora32/distro_test.go
View file

@ -121,13 +121,15 @@ func TestImageType_BuildPackages(t *testing.T) {
t.Errorf("d.GetArch(%v) returned err = %v; expected nil", archLabel, err)
continue
}
buildPkgs := itStruct.PackageSets(blueprint.Blueprint{})["build-packages"]
assert.NotNil(t, buildPkgs)
if itLabel == "fedora-iot-commit" {
// For now we only include rpm-ostree when building fedora-iot-commit image types, this we may want
// to reconsider. The only reason to specia-case it is that it might pull in a lot of dependencies
// for a niche usecase.
assert.ElementsMatch(t, append(buildPackages[archLabel], "rpm-ostree"), itStruct.BuildPackages())
assert.ElementsMatch(t, append(buildPackages[archLabel], "rpm-ostree"), buildPkgs.Include)
} else {
assert.ElementsMatch(t, buildPackages[archLabel], itStruct.BuildPackages())
assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs.Include)
}
}
}
@ -280,15 +282,16 @@ func TestImageType_BasePackages(t *testing.T) {
for _, pkgMap := range pkgMaps {
imgType, err := arch.GetImageType(pkgMap.name)
assert.NoError(t, err)
basePackages, excludedPackages := imgType.Packages(blueprint.Blueprint{})
packages := imgType.PackageSets(blueprint.Blueprint{})["packages"]
assert.NotNil(t, packages)
assert.Equalf(
t,
append(pkgMap.basePackages, pkgMap.bootloaderPackages...),
basePackages,
packages.Include,
"image type: %s",
pkgMap.name,
)
assert.Equalf(t, pkgMap.excludedPackages, excludedPackages, "image type: %s", pkgMap.name)
assert.Equalf(t, pkgMap.excludedPackages, packages.Exclude, "image type: %s", pkgMap.name)
}
}
@ -314,7 +317,7 @@ func TestDistro_ManifestError(t *testing.T) {
arch, _ := f32distro.GetArch(archName)
for _, imgTypeName := range arch.ListImageTypes() {
imgType, _ := arch.GetImageType(imgTypeName)
_, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, nil, 0)
_, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, 0)
if imgTypeName == "fedora-iot-commit" {
assert.EqualError(t, err, "kernel boot parameter customizations are not supported for ostree types")
} else {

20
internal/distro/fedora33/distro.go
View file

@ -187,20 +187,32 @@ func (t *imageType) BuildPackages() []string {
return packages
}
func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet {
includePackages, excludePackages := t.Packages(bp)
return map[string]rpmmd.PackageSet{
"packages": {
Include: includePackages,
Exclude: excludePackages,
},
"build-packages": {
Include: t.BuildPackages(),
},
}
}
func (t *imageType) Manifest(c *blueprint.Customizations,
options distro.ImageOptions,
repos []rpmmd.RepoConfig,
packageSpecs,
buildPackageSpecs []rpmmd.PackageSpec,
packageSpecSets map[string][]rpmmd.PackageSpec,
seed int64) (distro.Manifest, error) {
pipeline, err := t.pipeline(c, options, repos, packageSpecs, buildPackageSpecs)
pipeline, err := t.pipeline(c, options, repos, packageSpecSets["packages"], packageSpecSets["build-packages"])
if err != nil {
return distro.Manifest{}, err
}
return json.Marshal(
osbuild.Manifest{
Sources: *sources(append(packageSpecs, buildPackageSpecs...)),
Sources: *sources(append(packageSpecSets["packages"], packageSpecSets["build-packages"]...)),
Pipeline: *pipeline,
},
)

15
internal/distro/fedora33/distro_test.go
View file

@ -121,13 +121,15 @@ func TestImageType_BuildPackages(t *testing.T) {
t.Errorf("d.GetArch(%v) returned err = %v; expected nil", archLabel, err)
continue
}
buildPkgs := itStruct.PackageSets(blueprint.Blueprint{})["build-packages"]
assert.NotNil(t, buildPkgs)
if itLabel == "fedora-iot-commit" {
// For now we only include rpm-ostree when building fedora-iot-commit image types, this we may want
// to reconsider. The only reason to specia-case it is that it might pull in a lot of dependencies
// for a niche usecase.
assert.ElementsMatch(t, append(buildPackages[archLabel], "rpm-ostree"), itStruct.BuildPackages())
assert.ElementsMatch(t, append(buildPackages[archLabel], "rpm-ostree"), buildPkgs.Include)
} else {
assert.ElementsMatch(t, buildPackages[archLabel], itStruct.BuildPackages())
assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs.Include)
}
}
}
@ -286,15 +288,16 @@ func TestImageType_BasePackages(t *testing.T) {
for _, pkgMap := range pkgMaps {
imgType, err := arch.GetImageType(pkgMap.name)
assert.NoError(t, err)
basePackages, excludedPackages := imgType.Packages(blueprint.Blueprint{})
packages := imgType.PackageSets(blueprint.Blueprint{})["packages"]
assert.NotNil(t, packages)
assert.Equalf(
t,
append(pkgMap.basePackages, pkgMap.bootloaderPackages...),
basePackages,
packages.Include,
"image type: %s",
pkgMap.name,
)
assert.Equalf(t, pkgMap.excludedPackages, excludedPackages, "image type: %s", pkgMap.name)
assert.Equalf(t, pkgMap.excludedPackages, packages.Exclude, "image type: %s", pkgMap.name)
}
}
@ -320,7 +323,7 @@ func TestDistro_ManifestError(t *testing.T) {
arch, _ := f33distro.GetArch(archName)
for _, imgTypeName := range arch.ListImageTypes() {
imgType, _ := arch.GetImageType(imgTypeName)
_, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, nil, 0)
_, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, 0)
if imgTypeName == "fedora-iot-commit" {
assert.EqualError(t, err, "kernel boot parameter customizations are not supported for ostree types")
} else {

7
internal/distro/fedoratest/distro.go
View file

@ -95,11 +95,14 @@ func (t *imageType) BuildPackages() []string {
return nil
}
func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet {
return nil
}
func (t *imageType) Manifest(c *blueprint.Customizations,
options distro.ImageOptions,
repos []rpmmd.RepoConfig,
packageSpecs,
buildPackageSpecs []rpmmd.PackageSpec,
packageSpecSets map[string][]rpmmd.PackageSpec,
seed int64) (distro.Manifest, error) {
return json.Marshal(

20
internal/distro/rhel8/distro.go
View file

@ -188,20 +188,32 @@ func (t *imageType) BuildPackages() []string {
return packages
}
func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet {
includePackages, excludePackages := t.Packages(bp)
return map[string]rpmmd.PackageSet{
"packages": {
Include: includePackages,
Exclude: excludePackages,
},
"build-packages": {
Include: t.BuildPackages(),
},
}
}
func (t *imageType) Manifest(c *blueprint.Customizations,
options distro.ImageOptions,
repos []rpmmd.RepoConfig,
packageSpecs,
buildPackageSpecs []rpmmd.PackageSpec,
packageSpecSets map[string][]rpmmd.PackageSpec,
seed int64) (distro.Manifest, error) {
pipeline, err := t.pipeline(c, options, repos, packageSpecs, buildPackageSpecs)
pipeline, err := t.pipeline(c, options, repos, packageSpecSets["packages"], packageSpecSets["build-packages"])
if err != nil {
return distro.Manifest{}, err
}
return json.Marshal(
osbuild.Manifest{
Sources: *sources(append(packageSpecs, buildPackageSpecs...)),
Sources: *sources(append(packageSpecSets["packages"], packageSpecSets["build-packages"]...)),
Pipeline: *pipeline,
},
)

13
internal/distro/rhel8/distro_test.go
View file

@ -126,7 +126,9 @@ func TestImageType_BuildPackages(t *testing.T) {
if assert.NoErrorf(t, err, "d.GetArch(%v) returned err = %v; expected nil", archLabel, err) {
continue
}
assert.ElementsMatch(t, buildPackages[archLabel], itStruct.BuildPackages())
buildPkgs := itStruct.PackageSets(blueprint.Blueprint{})["build-packages"]
assert.NotNil(t, buildPkgs)
assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs.Include)
}
}
}
@ -339,15 +341,16 @@ func TestImageType_BasePackages(t *testing.T) {
for _, pkgMap := range pkgMaps {
imgType, err := arch.GetImageType(pkgMap.name)
assert.NoError(t, err)
basePackages, excludedPackages := imgType.Packages(blueprint.Blueprint{})
packages := imgType.PackageSets(blueprint.Blueprint{})["packages"]
assert.NotNil(t, packages)
assert.Equalf(
t,
append(pkgMap.basePackages, pkgMap.bootloaderPackages...),
basePackages,
packages.Include,
"image type: %s",
pkgMap.name,
)
assert.Equalf(t, pkgMap.excludedPackages, excludedPackages, "image type: %s", pkgMap.name)
assert.Equalf(t, pkgMap.excludedPackages, packages.Exclude, "image type: %s", pkgMap.name)
}
}
@ -372,7 +375,7 @@ func TestDistro_ManifestError(t *testing.T) {
arch, _ := r8distro.GetArch(archName)
for _, imgTypeName := range arch.ListImageTypes() {
imgType, _ := arch.GetImageType(imgTypeName)
_, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, nil, 0)
_, err := imgType.Manifest(bp.Customizations, distro.ImageOptions{}, nil, nil, 0)
if imgTypeName == "rhel-edge-commit" {
assert.EqualError(t, err, "kernel boot parameter customizations are not supported for ostree types")
} else {

20
internal/distro/rhel84/distro.go
View file

@ -207,22 +207,34 @@ func (t *imageType) BuildPackages() []string {
return packages
}
func (t *imageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet {
includePackages, excludePackages := t.Packages(bp)
return map[string]rpmmd.PackageSet{
"packages": {
Include: includePackages,
Exclude: excludePackages,
},
"build-packages": {
Include: t.BuildPackages(),
},
}
}
func (t *imageType) Manifest(c *blueprint.Customizations,
options distro.ImageOptions,
repos []rpmmd.RepoConfig,
packageSpecs,
buildPackageSpecs []rpmmd.PackageSpec,
packageSpecSets map[string][]rpmmd.PackageSpec,
seed int64) (distro.Manifest, error) {
source := rand.NewSource(seed)
rng := rand.New(source)
pipeline, err := t.pipeline(c, options, repos, packageSpecs, buildPackageSpecs, rng)
pipeline, err := t.pipeline(c, options, repos, packageSpecSets["packages"], packageSpecSets["build-packages"], rng)
if err != nil {
return distro.Manifest{}, err
}
return json.Marshal(
osbuild.Manifest{
Sources: *sources(append(packageSpecs, buildPackageSpecs...)),
Sources: *sources(append(packageSpecSets["packages"], packageSpecSets["build-packages"]...)),
Pipeline: *pipeline,
},
)

13
internal/distro/rhel84/distro_test.go
View file

@ -153,7 +153,9 @@ func TestImageType_BuildPackages(t *testing.T) {
if assert.NoErrorf(t, err, "d.GetArch(%v) returned err = %v; expected nil", archLabel, err) {
continue
}
assert.ElementsMatch(t, buildPackages[archLabel], itStruct.BuildPackages())
buildPkgs := itStruct.PackageSets(blueprint.Blueprint{})["build-packages"]
assert.NotNil(t, buildPkgs)
assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs.Include)
}
}
})
@ -393,7 +395,8 @@ func TestImageType_BasePackages(t *testing.T) {
for _, pkgMap := range pkgMaps {
imgType, err := arch.GetImageType(pkgMap.name)
assert.NoError(t, err)
basePackages, excludedPackages := imgType.Packages(blueprint.Blueprint{})
packages := imgType.PackageSets(blueprint.Blueprint{})["packages"]
assert.NotNil(t, packages)
expectedPackages := append(pkgMap.basePackages, pkgMap.bootloaderPackages...)
if dist.name == "rhel" {
expectedPackages = append(expectedPackages, pkgMap.rhelOnlyBasePackages...)
@ -401,11 +404,11 @@ func TestImageType_BasePackages(t *testing.T) {
assert.ElementsMatchf(
t,
expectedPackages,
basePackages,
packages.Include,
"image type: %s",
pkgMap.name,
)
assert.Equalf(t, pkgMap.excludedPackages, excludedPackages, "image type: %s", pkgMap.name)
assert.Equalf(t, pkgMap.excludedPackages, packages.Exclude, "image type: %s", pkgMap.name)
}
})
}
@ -442,7 +445,7 @@ func TestDistro_ManifestError(t *testing.T) {
imgOpts := distro.ImageOptions{
Size: imgType.Size(0),
}
_, err := imgType.Manifest(bp.Customizations, imgOpts, nil, nil, nil, 0)
_, err := imgType.Manifest(bp.Customizations, imgOpts, nil, nil, 0)
if imgTypeName == "rhel-edge-commit" {
assert.EqualError(t, err, "kernel boot parameter customizations are not supported for ostree types")
} else {

6
internal/distro/test_distro/distro.go
View file

@ -79,7 +79,11 @@ func (t *TestImageType) BuildPackages() []string {
return nil
}
func (t *TestImageType) Manifest(b *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSpecs, buildPackageSpecs []rpmmd.PackageSpec, seed int64) (distro.Manifest, error) {
func (t *TestImageType) PackageSets(bp blueprint.Blueprint) map[string]rpmmd.PackageSet {
return nil
}
func (t *TestImageType) Manifest(b *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSpecSets map[string][]rpmmd.PackageSpec, seed int64) (distro.Manifest, error) {
return json.Marshal(
osbuild.Manifest{
Sources: osbuild.Sources{},

20
internal/kojiapi/server.go
View file

@ -118,18 +118,18 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
if err != nil {
panic("Could not initialize empty blueprint.")
}
packageSpecs, excludePackageSpecs := imageType.Packages(*bp)
packages, _, err := h.server.rpmMetadata.Depsolve(packageSpecs, excludePackageSpecs, repositories, d.ModulePlatformID(), arch.Name())
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to depsolve base base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err))
}
buildPackageSpecs := imageType.BuildPackages()
buildPackages, _, err := h.server.rpmMetadata.Depsolve(buildPackageSpecs, nil, repositories, d.ModulePlatformID(), arch.Name())
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to depsolve build packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err))
packageSets := imageType.PackageSets(*bp)
packageSpecSets := make(map[string][]rpmmd.PackageSpec)
for name, packages := range packageSets {
packageSpecs, _, err := h.server.rpmMetadata.Depsolve(packages, repositories, d.ModulePlatformID(), arch.Name())
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to depsolve base base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err))
}
packageSpecSets[name] = packageSpecs
}
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, repositories, packages, buildPackages, manifestSeed)
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, repositories, packageSpecSets, manifestSeed)
if err != nil {
return echo.NewHTTPError(http.StatusBadGateway, fmt.Sprintf("Failed to get manifest for for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err))
}

2
internal/mocks/rpmmd/rpmmd_mock.go
View file

@ -36,6 +36,6 @@ func (r *rpmmdMock) FetchMetadata(repos []rpmmd.RepoConfig, modulePlatformID str
return r.Fixture.fetchPackageList.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.fetchPackageList.err
}
func (r *rpmmdMock) Depsolve(specs, excludeSpecs []string, repos []rpmmd.RepoConfig, modulePlatformID, arch string) ([]rpmmd.PackageSpec, map[string]string, error) {
func (r *rpmmdMock) Depsolve(packageSet rpmmd.PackageSet, repos []rpmmd.RepoConfig, modulePlatformID, arch string) ([]rpmmd.PackageSpec, map[string]string, error) {
return r.Fixture.depsolve.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.depsolve.err
}

15
internal/rpmmd/repository.go
View file

@ -96,6 +96,13 @@ func (pkg Package) ToPackageInfo() PackageInfo {
}
}
// The inputs to depsolve, a set of packages to include and a set of
// packages to exclude.
type PackageSet struct {
Include []string
Exclude []string
}
// TODO: the public API of this package should not be reused for serialization.
type PackageSpec struct {
Name string `json:"name"`
@ -159,7 +166,7 @@ type RPMMD interface {
// Depsolve takes a list of required content (specs), explicitly unwanted content (excludeSpecs), list
// or repositories, and platform ID for modularity. It returns a list of all packages (with solved
// dependencies) that will be installed into the system.
Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID, arch string) ([]PackageSpec, map[string]string, error)
Depsolve(packageSet PackageSet, repos []RepoConfig, modulePlatformID, arch string) ([]PackageSpec, map[string]string, error)
}
type DNFError struct {
@ -377,7 +384,7 @@ func (r *rpmmdImpl) FetchMetadata(repos []RepoConfig, modulePlatformID string, a
return reply.Packages, checksums, err
}
func (r *rpmmdImpl) Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID, arch string) ([]PackageSpec, map[string]string, error) {
func (r *rpmmdImpl) Depsolve(packageSet PackageSet, repos []RepoConfig, modulePlatformID, arch string) ([]PackageSpec, map[string]string, error) {
var dnfRepoConfigs []dnfRepoConfig
for i, repo := range repos {
@ -395,7 +402,7 @@ func (r *rpmmdImpl) Depsolve(specs, excludeSpecs []string, repos []RepoConfig, m
CacheDir string `json:"cachedir"`
ModulePlatformID string `json:"module_platform_id"`
Arch string `json:"arch"`
}{specs, excludeSpecs, dnfRepoConfigs, r.CacheDir, modulePlatformID, arch}
}{packageSet.Include, packageSet.Exclude, dnfRepoConfigs, r.CacheDir, modulePlatformID, arch}
var reply struct {
Checksums map[string]string `json:"checksums"`
Dependencies []dnfPackageSpec `json:"dependencies"`
@ -479,6 +486,6 @@ func (packages PackageList) ToPackageInfos() []PackageInfo {
}
func (pkg *PackageInfo) FillDependencies(rpmmd RPMMD, repos []RepoConfig, modulePlatformID string, arch string) (err error) {
pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, nil, repos, modulePlatformID, arch)
pkg.Dependencies, _, err = rpmmd.Depsolve(PackageSet{Include: []string{pkg.Name}}, repos, modulePlatformID, arch)
return
}

4
internal/store/fixtures.go
View file

@ -57,7 +57,7 @@ func FixtureBase() *Store {
if err != nil {
panic("invalid image type qcow2 for x86_64 @ fedoratest")
}
manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, nil, 0)
manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, 0)
if err != nil {
panic("could not create manifest")
}
@ -160,7 +160,7 @@ func FixtureFinished() *Store {
if err != nil {
panic("invalid image type qcow2 for x86_64 @ fedoratest")
}
manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, nil, 0)
manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, 0)
if err != nil {
panic("could not create manifest")
}

2
internal/store/store_test.go
View file

@ -51,7 +51,7 @@ func (suite *storeTest) SetupSuite() {
suite.myDistro = test_distro.New()
suite.myArch, _ = suite.myDistro.GetArch("test_arch")
suite.myImageType, _ = suite.myArch.GetImageType("test_type")
suite.myManifest, _ = suite.myImageType.Manifest(&suite.myCustomizations, suite.myImageOptions, suite.myRepoConfig, nil, suite.myPackageSpec, 0)
suite.myManifest, _ = suite.myImageType.Manifest(&suite.myCustomizations, suite.myImageOptions, suite.myRepoConfig, nil, 0)
suite.mySourceConfig = SourceConfig{
Name: "testSourceConfig",
}

51
internal/weldr/api.go
View file

@ -1038,7 +1038,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt
projects = projects[1:]
names := strings.Split(projects, ",")
packages, _, err := api.rpmmd.Depsolve(names, nil, api.repos, api.distro.ModulePlatformID(), api.arch.Name())
packages, _, err := api.rpmmd.Depsolve(rpmmd.PackageSet{Include: names}, api.repos, api.distro.ModulePlatformID(), api.arch.Name())
if err != nil {
errors := responseError{
@ -1233,7 +1233,7 @@ func (api *API) blueprintsDepsolveHandler(writer http.ResponseWriter, request *h
continue
}
dependencies, _, err := api.depsolveBlueprint(blueprint, nil)
dependencies, err := api.depsolveBlueprint(blueprint)
if err != nil {
blueprintsErrors = append(blueprintsErrors, responseError{
@ -1322,7 +1322,7 @@ func (api *API) blueprintsFreezeHandler(writer http.ResponseWriter, request *htt
}
// Make a copy of the blueprint since we will be replacing the version globs
blueprint := bp.DeepCopy()
dependencies, _, err := api.depsolveBlueprint(&blueprint, nil)
dependencies, err := api.depsolveBlueprint(&blueprint)
if err != nil {
rerr := responseError{
ID: "BlueprintsError",
@ -1829,6 +1829,19 @@ func ostreeResolveRef(location, ref string) (string, error) {
return parent, nil
}
func (api *API) depsolveBlueprintForImageType(bp *blueprint.Blueprint, imageType distro.ImageType) (map[string][]rpmmd.PackageSpec, error) {
packageSets := imageType.PackageSets(*bp)
packageSpecSets := make(map[string][]rpmmd.PackageSpec)
for name, packageSet := range packageSets {
packageSpecs, _, err := api.rpmmd.Depsolve(packageSet, api.allRepositories(), api.distro.ModulePlatformID(), api.arch.Name())
if err != nil {
return nil, err
}
packageSpecSets[name] = packageSpecs
}
return packageSpecSets, nil
}
// Schedule new compose by first translating the appropriate blueprint into a pipeline and then
// pushing it into the channel for waiting builds.
func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {
@ -1959,7 +1972,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
cr.OSTree.Parent = parent
}
packages, buildPackages, err := api.depsolveBlueprint(bp, imageType)
packageSets, err := api.depsolveBlueprintForImageType(bp, imageType)
if err != nil {
errors := responseError{
ID: "DepsolveError",
@ -1986,8 +1999,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
},
},
api.allRepositories(),
packages,
buildPackages,
packageSets,
seed)
if err != nil {
errors := responseError{
@ -2738,32 +2750,13 @@ func (api *API) allRepositories() []rpmmd.RepoConfig {
return repos
}
func (api *API) depsolveBlueprint(bp *blueprint.Blueprint, imageType distro.ImageType) ([]rpmmd.PackageSpec, []rpmmd.PackageSpec, error) {
repos := api.allRepositories()
specs := bp.GetPackages()
excludeSpecs := []string{}
if imageType != nil {
// When the output type is known, include the base packages in the depsolve
// transaction.
specs, excludeSpecs = imageType.Packages(*bp)
}
packages, _, err := api.rpmmd.Depsolve(specs, excludeSpecs, repos, api.distro.ModulePlatformID(), api.arch.Name())
func (api *API) depsolveBlueprint(bp *blueprint.Blueprint) ([]rpmmd.PackageSpec, error) {
packages, _, err := api.rpmmd.Depsolve(rpmmd.PackageSet{Include: bp.GetPackages()}, api.allRepositories(), api.distro.ModulePlatformID(), api.arch.Name())
if err != nil {
return nil, nil, err
return nil, err
}
buildPackages := []rpmmd.PackageSpec{}
if imageType != nil {
buildSpecs := imageType.BuildPackages()
buildPackages, _, err = api.rpmmd.Depsolve(buildSpecs, nil, repos, api.distro.ModulePlatformID(), api.arch.Name())
if err != nil {
return nil, nil, err
}
}
return packages, buildPackages, err
return packages, err
}
func (api *API) uploadsScheduleHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {

2
internal/weldr/api_test.go
View file

@ -578,7 +578,7 @@ func TestCompose(t *testing.T) {
require.NoError(t, err)
imgType, err := arch.GetImageType("qcow2")
require.NoError(t, err)
manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, nil, 0)
manifest, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, nil, 0)
require.NoError(t, err)
expectedComposeLocal := &store.Compose{
Blueprint: &blueprint.Blueprint{

10
internal/worker/server_test.go
View file

@ -83,7 +83,7 @@ func TestCreate(t *testing.T) {
if err != nil {
t.Fatalf("error getting image type from arch")
}
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0)
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0)
if err != nil {
t.Fatalf("error creating osbuild manifest")
}
@ -111,7 +111,7 @@ func TestCancel(t *testing.T) {
if err != nil {
t.Fatalf("error getting image type from arch")
}
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0)
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0)
if err != nil {
t.Fatalf("error creating osbuild manifest")
}
@ -152,7 +152,7 @@ func TestUpdate(t *testing.T) {
if err != nil {
t.Fatalf("error getting image type from arch")
}
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0)
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0)
if err != nil {
t.Fatalf("error creating osbuild manifest")
}
@ -179,7 +179,7 @@ func TestArgs(t *testing.T) {
require.NoError(t, err)
imageType, err := arch.GetImageType("qcow2")
require.NoError(t, err)
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0)
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0)
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "worker-tests-")
@ -221,7 +221,7 @@ func TestUpload(t *testing.T) {
if err != nil {
t.Fatalf("error getting image type from arch")
}
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, nil, 0)
manifest, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, nil, 0)
if err != nil {
t.Fatalf("error creating osbuild manifest")
}