particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

distro/fedora/PackageSets: get from Manifest object

Browse source 
Before you read this patch, be warned: this is crazy. But rest
assured, this is as bad as it gets, and by the end of this PR it
will all be infinitely simpler than it ever was.

We want Manifests to be self-describing, and in particular, we want
manifest.GetPackageSetChains() to return what
ImageType.GetPackages does today. This should be achieved by
pipelines knowing their required package sets, and the caller
optionally amending additional package sets at Pipeline
initialisation time.

As a first step, while pipelines don't yet know anything about their
required package sets, simply pass in the precomputed
PackageSetChains and set them as optional on each pipeline. Then
we can read the chains back out of t he manifest.

This is not a functional change, but allows us to gradually move
package set definitions from the distros to the pipelines in follow
ups.
This commit is contained in:
Tom Gundersen 2022-06-29 14:44:28 +01:00
parent be5ea6a9b8
commit c042af7d9c
5 changed files with 181 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

23
internal/distro/distro_test_common/distro_test_common.go
View file

@ -201,34 +201,15 @@ func kernelCount(imgType distro.ImageType) int {
}
}
// TODO: is this not included in the above?
for _, bset := range sets["blueprint"] {
for _, pkg := range bset.Include {
for _, iset := range sets["installer"] {
for _, pkg := range iset.Include {
for _, kernel := range knownKernels {
if kernel == pkg {
n++
}
}
}
}
m := 0
for _, iset := range sets["installer"] {
for _, pkg := range iset.Include {
for _, kernel := range knownKernels {
if kernel == pkg {
m++
}
}
}
}
// TODO: fix this! some installers mistakenly include other
// package sets. These are however not used in building the image
// until this is fixed, count the installers separately.
if m > 0 {
return m
}
return n
}

59
internal/distro/fedora/distro.go
View file

@ -490,7 +490,7 @@ func (a *architecture) Distro() distro.Distro {
return a.distro
}
type pipelinesFunc func(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error)
type pipelinesFunc func(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error)
type packageSetFunc func(t *imageType) rpmmd.PackageSet
@ -626,8 +626,38 @@ func (t *imageType) PackageSets(bp blueprint.Blueprint, options distro.ImageOpti
if t.rpmOstree || t.bootable {
mergedSets[osPkgsKey] = mergedSets[osPkgsKey].Append(rpmmd.PackageSet{Include: []string{kernel}})
}
return distro.MakePackageSetChains(t, mergedSets, repos)
// create a manifest object and instantiate it with the computed packageSetChains
manifest, err := t.initializeManifest(bp.Customizations, options, repos, distro.MakePackageSetChains(t, mergedSets, repos), nil, 0)
if err != nil {
// TODO: handle manifest initialization errors more gracefully, we
// refuse to initialize manifests with invalid config.
return nil
}
manifestChains := manifest.GetPackageSetChains()
// the returned package set chains are indexed by pipeline
// name, we need to reindex by package set name
distroChains := make(map[string][]rpmmd.PackageSet)
for name, chain := range manifestChains {
switch name {
case "os":
name = osPkgsKey
case "ostree-tree":
name = osPkgsKey
case "container-tree":
name = containerPkgsKey
case "anaconda-tree":
name = installerPkgsKey
case "build":
name = buildPkgsKey
default:
panic(fmt.Sprintf("unknown pacakge set name: %s", name))
}
distroChains[name] = chain
}
return distroChains
}
func (t *imageType) BuildPipelines() []string {
@ -704,14 +734,15 @@ func (t *imageType) PartitionType() string {
return basePartitionTable.Type
}
func (t *imageType) Manifest(customizations *blueprint.Customizations,
func (t *imageType) initializeManifest(customizations *blueprint.Customizations,
options distro.ImageOptions,
repos []rpmmd.RepoConfig,
packageSetChains map[string][]rpmmd.PackageSet,
packageSpecSets map[string][]rpmmd.PackageSpec,
seed int64) (distro.Manifest, error) {
seed int64) (*manifest.Manifest, error) {
if err := t.checkOptions(customizations, options); err != nil {
return distro.Manifest{}, err
return nil, err
}
source := rand.NewSource(seed)
@ -719,9 +750,9 @@ func (t *imageType) Manifest(customizations *blueprint.Customizations,
/* #nosec G404 */
rng := rand.New(source)
pipelines, err := t.pipelines(t, customizations, options, repos, packageSpecSets, rng)
pipelines, err := t.pipelines(t, customizations, options, repos, packageSetChains, packageSpecSets, rng)
if err != nil {
return distro.Manifest{}, err
return nil, err
}
manifest := manifest.New()
@ -730,6 +761,20 @@ func (t *imageType) Manifest(customizations *blueprint.Customizations,
manifest.AddPipeline(pipeline)
}
return &manifest, nil
}
func (t *imageType) Manifest(customizations *blueprint.Customizations,
options distro.ImageOptions,
repos []rpmmd.RepoConfig,
packageSpecSets map[string][]rpmmd.PackageSpec,
seed int64) (distro.Manifest, error) {
manifest, err := t.initializeManifest(customizations, options, repos, nil, packageSpecSets, seed)
if err != nil {
return distro.Manifest{}, err
}
return manifest.Serialize()
}

137
internal/distro/fedora/pipelines.go
View file

@ -10,7 +10,7 @@ import (
"github.com/osbuild/osbuild-composer/internal/rpmmd"
)
func qcow2Pipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
func qcow2Pipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
pipelines := make([]manifest.Pipeline, 0)
buildPipeline := manifest.NewBuildPipeline(t.arch.distro.runner, repos, packageSetSpecs[buildPkgsKey])
@ -20,6 +20,18 @@ func qcow2Pipelines(t *imageType, customizations *blueprint.Customizations, opti
if err != nil {
return nil, err
}
osChain := packageSetChains[osPkgsKey]
if len(osChain) >= 1 {
treePipeline.ExtraBasePackages = osChain[0].Include
treePipeline.ExcludeBasePackages = osChain[0].Exclude
}
if len(osChain) >= 2 {
treePipeline.UserPackages = osChain[0].Include
treePipeline.UserRepos = osChain[0].Repositories
}
if len(osChain) > 2 {
panic("unexpected number of package sets in os chain")
}
pipelines = append(pipelines, treePipeline)
imagePipeline := manifest.NewLiveImgPipeline(buildPipeline, treePipeline, "disk.img")
@ -32,7 +44,7 @@ func qcow2Pipelines(t *imageType, customizations *blueprint.Customizations, opti
return pipelines, nil
}
func vhdPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
func vhdPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
pipelines := make([]manifest.Pipeline, 0)
buildPipeline := manifest.NewBuildPipeline(t.arch.distro.runner, repos, packageSetSpecs[buildPkgsKey])
@ -42,6 +54,18 @@ func vhdPipelines(t *imageType, customizations *blueprint.Customizations, option
if err != nil {
return nil, err
}
osChain := packageSetChains[osPkgsKey]
if len(osChain) >= 1 {
treePipeline.ExtraBasePackages = osChain[0].Include
treePipeline.ExcludeBasePackages = osChain[0].Exclude
}
if len(osChain) >= 2 {
treePipeline.UserPackages = osChain[0].Include
treePipeline.UserRepos = osChain[0].Repositories
}
if len(osChain) > 2 {
panic("unexpected number of package sets in os chain")
}
pipelines = append(pipelines, treePipeline)
imagePipeline := manifest.NewLiveImgPipeline(buildPipeline, treePipeline, "disk.img")
@ -52,7 +76,7 @@ func vhdPipelines(t *imageType, customizations *blueprint.Customizations, option
return pipelines, nil
}
func vmdkPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
func vmdkPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
pipelines := make([]manifest.Pipeline, 0)
buildPipeline := manifest.NewBuildPipeline(t.arch.distro.runner, repos, packageSetSpecs[buildPkgsKey])
@ -62,6 +86,18 @@ func vmdkPipelines(t *imageType, customizations *blueprint.Customizations, optio
if err != nil {
return nil, err
}
osChain := packageSetChains[osPkgsKey]
if len(osChain) >= 1 {
treePipeline.ExtraBasePackages = osChain[0].Include
treePipeline.ExcludeBasePackages = osChain[0].Exclude
}
if len(osChain) >= 2 {
treePipeline.UserPackages = osChain[0].Include
treePipeline.UserRepos = osChain[0].Repositories
}
if len(osChain) > 2 {
panic("unexpected number of package sets in os chain")
}
pipelines = append(pipelines, treePipeline)
imagePipeline := manifest.NewLiveImgPipeline(buildPipeline, treePipeline, "disk.img")
@ -72,7 +108,7 @@ func vmdkPipelines(t *imageType, customizations *blueprint.Customizations, optio
return pipelines, nil
}
func openstackPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
func openstackPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
pipelines := make([]manifest.Pipeline, 0)
buildPipeline := manifest.NewBuildPipeline(t.arch.distro.runner, repos, packageSetSpecs[buildPkgsKey])
@ -82,6 +118,18 @@ func openstackPipelines(t *imageType, customizations *blueprint.Customizations,
if err != nil {
return nil, err
}
osChain := packageSetChains[osPkgsKey]
if len(osChain) >= 1 {
treePipeline.ExtraBasePackages = osChain[0].Include
treePipeline.ExcludeBasePackages = osChain[0].Exclude
}
if len(osChain) >= 2 {
treePipeline.UserPackages = osChain[0].Include
treePipeline.UserRepos = osChain[0].Repositories
}
if len(osChain) > 2 {
panic("unexpected number of package sets in os chain")
}
pipelines = append(pipelines, treePipeline)
imagePipeline := manifest.NewLiveImgPipeline(buildPipeline, treePipeline, "disk.img")
@ -93,7 +141,8 @@ func openstackPipelines(t *imageType, customizations *blueprint.Customizations,
}
func ec2CommonPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions,
repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec,
repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet,
packageSetSpecs map[string][]rpmmd.PackageSpec,
rng *rand.Rand, diskfile string) ([]manifest.Pipeline, error) {
pipelines := make([]manifest.Pipeline, 0)
@ -104,6 +153,18 @@ func ec2CommonPipelines(t *imageType, customizations *blueprint.Customizations,
if err != nil {
return nil, err
}
osChain := packageSetChains[osPkgsKey]
if len(osChain) >= 1 {
treePipeline.ExtraBasePackages = osChain[0].Include
treePipeline.ExcludeBasePackages = osChain[0].Exclude
}
if len(osChain) >= 2 {
treePipeline.UserPackages = osChain[0].Include
treePipeline.UserRepos = osChain[0].Repositories
}
if len(osChain) > 2 {
panic("unexpected number of package sets in os chain")
}
pipelines = append(pipelines, treePipeline)
imagePipeline := manifest.NewLiveImgPipeline(buildPipeline, treePipeline, diskfile)
@ -112,11 +173,15 @@ func ec2CommonPipelines(t *imageType, customizations *blueprint.Customizations,
}
// ec2Pipelines returns pipelines which produce uncompressed EC2 images which are expected to use RHSM for content
func ec2Pipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
return ec2CommonPipelines(t, customizations, options, repos, packageSetSpecs, rng, t.Filename())
func ec2Pipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions,
repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet,
packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
return ec2CommonPipelines(t, customizations, options, repos, packageSetChains, packageSetSpecs, rng, t.Filename())
}
func iotInstallerPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
func iotInstallerPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions,
repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet, packageSetSpecs map[string][]rpmmd.PackageSpec,
rng *rand.Rand) ([]manifest.Pipeline, error) {
pipelines := make([]manifest.Pipeline, 0)
buildPipeline := manifest.NewBuildPipeline(t.arch.distro.runner, repos, packageSetSpecs[buildPkgsKey])
@ -127,27 +192,50 @@ func iotInstallerPipelines(t *imageType, customizations *blueprint.Customization
ksUsers := len(customizations.GetUsers())+len(customizations.GetGroups()) > 0
anacondaTreePipeline := anacondaTreePipeline(buildPipeline, repos, installerPackages, t.Arch().Name(), d.product, d.osVersion, "IoT", ksUsers)
installerChain := packageSetChains[installerPkgsKey]
if len(installerChain) >= 1 {
anacondaTreePipeline.ExtraPackages = installerChain[0].Include
}
if len(installerChain) > 1 {
panic("unexpected number of package sets in installer chain")
}
isoTreePipeline := bootISOTreePipeline(buildPipeline, anacondaTreePipeline, options, d.vendor, d.isolabelTmpl, customizations.GetUsers(), customizations.GetGroups())
isoPipeline := bootISOPipeline(buildPipeline, isoTreePipeline, t.Filename(), false)
return append(pipelines, anacondaTreePipeline, isoTreePipeline, isoPipeline), nil
}
func iotCorePipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec) (*manifest.BuildPipeline, *manifest.OSPipeline, *manifest.OSTreeCommitPipeline, error) {
func iotCorePipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions,
repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet,
packageSetSpecs map[string][]rpmmd.PackageSpec) (*manifest.BuildPipeline, *manifest.OSPipeline, *manifest.OSTreeCommitPipeline, error) {
buildPipeline := manifest.NewBuildPipeline(t.arch.distro.runner, repos, packageSetSpecs[buildPkgsKey])
treePipeline, err := osPipeline(buildPipeline, t, repos, packageSetSpecs[osPkgsKey], customizations, options, nil)
if err != nil {
return nil, nil, nil, err
}
osChain := packageSetChains[osPkgsKey]
if len(osChain) >= 1 {
treePipeline.ExtraBasePackages = osChain[0].Include
treePipeline.ExcludeBasePackages = osChain[0].Exclude
}
if len(osChain) >= 2 {
treePipeline.UserPackages = osChain[0].Include
treePipeline.UserRepos = osChain[0].Repositories
}
if len(osChain) > 2 {
panic("unexpected number of package sets in os chain")
}
commitPipeline := ostreeCommitPipeline(buildPipeline, treePipeline, options, t.arch.distro.osVersion)
return buildPipeline, treePipeline, commitPipeline, nil
}
func iotCommitPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
func iotCommitPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions,
repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet,
packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
pipelines := make([]manifest.Pipeline, 0)
buildPipeline, treePipeline, commitPipeline, err := iotCorePipelines(t, customizations, options, repos, packageSetSpecs)
buildPipeline, treePipeline, commitPipeline, err := iotCorePipelines(t, customizations, options, repos, packageSetChains, packageSetSpecs)
if err != nil {
return nil, err
}
@ -156,10 +244,12 @@ func iotCommitPipelines(t *imageType, customizations *blueprint.Customizations,
return pipelines, nil
}
func iotContainerPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
func iotContainerPipelines(t *imageType, customizations *blueprint.Customizations,
options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet,
packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
pipelines := make([]manifest.Pipeline, 0)
buildPipeline, treePipeline, commitPipeline, err := iotCorePipelines(t, customizations, options, repos, packageSetSpecs)
buildPipeline, treePipeline, commitPipeline, err := iotCorePipelines(t, customizations, options, repos, packageSetChains, packageSetSpecs)
if err != nil {
return nil, err
}
@ -167,6 +257,13 @@ func iotContainerPipelines(t *imageType, customizations *blueprint.Customization
nginxConfigPath := "/etc/nginx.conf"
httpPort := "8080"
containerTreePipeline := containerTreePipeline(buildPipeline, commitPipeline, repos, packageSetSpecs[containerPkgsKey], options, customizations, nginxConfigPath, httpPort)
containerChain := packageSetChains[osPkgsKey]
if len(containerChain) >= 1 {
containerTreePipeline.ExtraPackages = containerChain[0].Include
}
if len(containerChain) > 2 {
panic("unexpected number of package sets in os chain")
}
containerPipeline := containerPipeline(buildPipeline, &containerTreePipeline.BasePipeline, t, nginxConfigPath, httpPort)
pipelines = append(pipelines, buildPipeline, treePipeline, commitPipeline, containerTreePipeline, containerPipeline)
@ -346,7 +443,7 @@ func bootISOPipeline(buildPipeline *manifest.BuildPipeline, treePipeline *manife
return p
}
func containerPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
func containerPipelines(t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSetChains map[string][]rpmmd.PackageSet, packageSetSpecs map[string][]rpmmd.PackageSpec, rng *rand.Rand) ([]manifest.Pipeline, error) {
pipelines := make([]manifest.Pipeline, 0)
buildPipeline := manifest.NewBuildPipeline(t.arch.distro.runner, repos, packageSetSpecs[buildPkgsKey])
@ -356,6 +453,18 @@ func containerPipelines(t *imageType, customizations *blueprint.Customizations,
if err != nil {
return nil, err
}
osChain := packageSetChains[osPkgsKey]
if len(osChain) >= 1 {
treePipeline.ExtraBasePackages = osChain[0].Include
treePipeline.ExcludeBasePackages = osChain[0].Exclude
}
if len(osChain) >= 2 {
treePipeline.UserPackages = osChain[0].Include
treePipeline.UserRepos = osChain[0].Repositories
}
if len(osChain) > 2 {
panic("unexpected number of package sets in os chain")
}
pipelines = append(pipelines, treePipeline)
ociPipeline := manifest.NewOCIContainerPipeline(buildPipeline, &treePipeline.BasePipeline, t.Arch().Name(), t.Filename())

4
internal/manifest/manifest.go
View file

@ -62,7 +62,9 @@ func (m Manifest) GetPackageSetChains() map[string][]rpmmd.PackageSet {
chains := make(map[string][]rpmmd.PackageSet)
for _, pipeline := range m.pipelines {
chains[pipeline.Name()] = pipeline.getPackageSetChain()
if chain := pipeline.getPackageSetChain(); chain != nil {
chains[pipeline.Name()] = chain
}
}
return chains

2
internal/manifest/pipeline.go
View file

@ -43,7 +43,7 @@ func (p BasePipeline) getBuildPackages() []string {
}
func (p BasePipeline) getPackageSetChain() []rpmmd.PackageSet {
return []rpmmd.PackageSet{}
return nil
}
func (p BasePipeline) getPackageSpecs() []rpmmd.PackageSpec {