From 7a534d4d1e3bbccc1adbc7c5b0996f6deba9fbf8 Mon Sep 17 00:00:00 2001 From: Tom Gundersen Date: Wed, 6 Jul 2022 11:26:51 +0100 Subject: [PATCH] workload: introduce abstraction to encapsulate image workloads The workload encapsulates what the user wants to run on top of the image. Everything else we do abstracts away the OS, the hardware, the environment, and what is left is what matters: the workload. For now only the `Custom` payload is implemented which requires the user to name the packages they want installed, the repositories to pull them from and what systemd services to enable. A few other stub workloads are added to show the idea, but these are not used. The ideal is for the workload to have only the minimal number of configuration options. --- internal/distro/fedora/distro.go | 38 ++++++++++---- internal/distro/fedora/manifests.go | 73 ++++++++++++++++----------- internal/manifest/os.go | 28 +++++----- internal/workload/anaconda.go | 7 +++ internal/workload/custom.go | 22 ++++++++ internal/workload/sap.go | 6 +++ internal/workload/static_webserver.go | 7 +++ internal/workload/workload.go | 30 +++++++++++ 8 files changed, 159 insertions(+), 52 deletions(-) create mode 100644 internal/workload/anaconda.go create mode 100644 internal/workload/custom.go create mode 100644 internal/workload/sap.go create mode 100644 internal/workload/static_webserver.go create mode 100644 internal/workload/workload.go diff --git a/internal/distro/fedora/distro.go b/internal/distro/fedora/distro.go index fb5589b26..63cc21006 100644 --- a/internal/distro/fedora/distro.go +++ b/internal/distro/fedora/distro.go @@ -13,6 +13,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/manifest" "github.com/osbuild/osbuild-composer/internal/platform" "github.com/osbuild/osbuild-composer/internal/rpmmd" + "github.com/osbuild/osbuild-composer/internal/workload" ) const ( @@ -452,7 +453,7 @@ func (a *architecture) Distro() distro.Distro { return a.distro } -type manifestFunc func(m *manifest.Manifest, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSets map[string]rpmmd.PackageSet, rng *rand.Rand) error +type manifestFunc func(m *manifest.Manifest, workload workload.Workload, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSets map[string]rpmmd.PackageSet, rng *rand.Rand) error type packageSetFunc func(t *imageType) rpmmd.PackageSet @@ -527,11 +528,6 @@ func (t *imageType) PackageSets(bp blueprint.Blueprint, options distro.ImageOpti packageSets[name] = getter(t) } - // depsolve bp packages separately - // bp packages aren't restricted by exclude lists - // do not include the kernel package as it is included from the pipeline - packageSets[blueprintPkgsKey] = rpmmd.PackageSet{Include: bp.GetPackagesEx(false)} - // amend with repository information globalRepos := make([]rpmmd.RepoConfig, 0) for _, repo := range repos { @@ -550,7 +546,7 @@ func (t *imageType) PackageSets(bp blueprint.Blueprint, options distro.ImageOpti } // create a manifest object and instantiate it with the computed packageSetChains - manifest, err := t.initializeManifest(bp.Customizations, options, globalRepos, packageSets, 0) + manifest, err := t.initializeManifest(&bp, options, globalRepos, packageSets, 0) if err != nil { // TODO: handle manifest initialization errors more gracefully, we // refuse to initialize manifests with invalid config. @@ -619,23 +615,36 @@ func (t *imageType) PartitionType() string { return basePartitionTable.Type } -func (t *imageType) initializeManifest(customizations *blueprint.Customizations, +func (t *imageType) initializeManifest(bp *blueprint.Blueprint, options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSets map[string]rpmmd.PackageSet, seed int64) (*manifest.Manifest, error) { - if err := t.checkOptions(customizations, options); err != nil { + if err := t.checkOptions(bp.Customizations, options); err != nil { return nil, err } + // TODO: let image types specify valid workloads, rather than + // always assume Custom. + w := &workload.Custom{ + BaseWorkload: workload.BaseWorkload{ + Repos: packageSets[blueprintPkgsKey].Repositories, + }, + Packages: bp.GetPackagesEx(false), + } + if services := bp.Customizations.GetServices(); services != nil { + w.Services = services.Enabled + w.DisabledServices = services.Disabled + } + source := rand.NewSource(seed) // math/rand is good enough in this case /* #nosec G404 */ rng := rand.New(source) manifest := manifest.New() - err := t.manifest(&manifest, t, customizations, options, repos, packageSets, rng) + err := t.manifest(&manifest, w, t, bp.Customizations, options, repos, packageSets, rng) if err != nil { return nil, err } @@ -649,7 +658,14 @@ func (t *imageType) Manifest(customizations *blueprint.Customizations, packageSets map[string][]rpmmd.PackageSpec, seed int64) (distro.Manifest, error) { - manifest, err := t.initializeManifest(customizations, options, repos, nil, seed) + bp := &blueprint.Blueprint{} + err := bp.Initialize() + if err != nil { + panic("could not initialize empty blueprint: " + err.Error()) + } + bp.Customizations = customizations + + manifest, err := t.initializeManifest(bp, options, repos, nil, seed) if err != nil { return distro.Manifest{}, err } diff --git a/internal/distro/fedora/manifests.go b/internal/distro/fedora/manifests.go index 9f02d817c..b0b297933 100644 --- a/internal/distro/fedora/manifests.go +++ b/internal/distro/fedora/manifests.go @@ -7,9 +7,11 @@ import ( "github.com/osbuild/osbuild-composer/internal/distro" "github.com/osbuild/osbuild-composer/internal/manifest" "github.com/osbuild/osbuild-composer/internal/rpmmd" + "github.com/osbuild/osbuild-composer/internal/workload" ) func qcow2Manifest(m *manifest.Manifest, + workload workload.Workload, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, @@ -18,7 +20,7 @@ func qcow2Manifest(m *manifest.Manifest, rng *rand.Rand) error { buildPipeline := manifest.NewBuildPipeline(m, t.arch.distro.runner, repos) - treePipeline, err := osPipeline(m, buildPipeline, t, repos, packageSets[osPkgsKey], packageSets[blueprintPkgsKey], customizations, options, rng) + treePipeline, err := osPipeline(m, buildPipeline, workload, t, repos, packageSets[osPkgsKey], customizations, options, rng) if err != nil { return err } @@ -30,6 +32,7 @@ func qcow2Manifest(m *manifest.Manifest, } func vhdManifest(m *manifest.Manifest, + workload workload.Workload, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, @@ -38,7 +41,7 @@ func vhdManifest(m *manifest.Manifest, rng *rand.Rand) error { buildPipeline := manifest.NewBuildPipeline(m, t.arch.distro.runner, repos) - treePipeline, err := osPipeline(m, buildPipeline, t, repos, packageSets[osPkgsKey], packageSets[blueprintPkgsKey], customizations, options, rng) + treePipeline, err := osPipeline(m, buildPipeline, workload, t, repos, packageSets[osPkgsKey], customizations, options, rng) if err != nil { return err } @@ -49,6 +52,7 @@ func vhdManifest(m *manifest.Manifest, } func vmdkManifest(m *manifest.Manifest, + workload workload.Workload, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, @@ -57,7 +61,7 @@ func vmdkManifest(m *manifest.Manifest, rng *rand.Rand) error { buildPipeline := manifest.NewBuildPipeline(m, t.arch.distro.runner, repos) - treePipeline, err := osPipeline(m, buildPipeline, t, repos, packageSets[osPkgsKey], packageSets[blueprintPkgsKey], customizations, options, rng) + treePipeline, err := osPipeline(m, buildPipeline, workload, t, repos, packageSets[osPkgsKey], customizations, options, rng) if err != nil { return err } @@ -68,6 +72,7 @@ func vmdkManifest(m *manifest.Manifest, } func openstackManifest(m *manifest.Manifest, + workload workload.Workload, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, @@ -76,7 +81,7 @@ func openstackManifest(m *manifest.Manifest, rng *rand.Rand) error { buildPipeline := manifest.NewBuildPipeline(m, t.arch.distro.runner, repos) - treePipeline, err := osPipeline(m, buildPipeline, t, repos, packageSets[osPkgsKey], packageSets[blueprintPkgsKey], customizations, options, rng) + treePipeline, err := osPipeline(m, buildPipeline, workload, t, repos, packageSets[osPkgsKey], customizations, options, rng) if err != nil { return err } @@ -87,6 +92,7 @@ func openstackManifest(m *manifest.Manifest, } func ec2CommonManifest(m *manifest.Manifest, + workload workload.Workload, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, @@ -96,7 +102,7 @@ func ec2CommonManifest(m *manifest.Manifest, diskfile string) error { buildPipeline := manifest.NewBuildPipeline(m, t.arch.distro.runner, repos) - treePipeline, err := osPipeline(m, buildPipeline, t, repos, packageSets[osPkgsKey], packageSets[blueprintPkgsKey], customizations, options, rng) + treePipeline, err := osPipeline(m, buildPipeline, workload, t, repos, packageSets[osPkgsKey], customizations, options, rng) if err != nil { return nil } @@ -106,13 +112,19 @@ func ec2CommonManifest(m *manifest.Manifest, } // ec2Manifest returns a manifest which produce uncompressed EC2 images which are expected to use RHSM for content -func ec2Manifest(m *manifest.Manifest, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, - repos []rpmmd.RepoConfig, packageSets map[string]rpmmd.PackageSet, +func ec2Manifest(m *manifest.Manifest, + workload workload.Workload, + t *imageType, + customizations *blueprint.Customizations, + options distro.ImageOptions, + repos []rpmmd.RepoConfig, + packageSets map[string]rpmmd.PackageSet, rng *rand.Rand) error { - return ec2CommonManifest(m, t, customizations, options, repos, packageSets, rng, t.Filename()) + return ec2CommonManifest(m, workload, t, customizations, options, repos, packageSets, rng, t.Filename()) } func iotInstallerManifest(m *manifest.Manifest, + workload workload.Workload, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, @@ -133,6 +145,7 @@ func iotInstallerManifest(m *manifest.Manifest, } func iotCorePipelines(m *manifest.Manifest, + workload workload.Workload, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, @@ -141,7 +154,7 @@ func iotCorePipelines(m *manifest.Manifest, *manifest.OSTreeCommitPipeline, error) { buildPipeline := manifest.NewBuildPipeline(m, t.arch.distro.runner, repos) - treePipeline, err := osPipeline(m, buildPipeline, t, repos, packageSets[osPkgsKey], packageSets[blueprintPkgsKey], customizations, options, nil) + treePipeline, err := osPipeline(m, buildPipeline, workload, t, repos, packageSets[osPkgsKey], customizations, options, nil) if err != nil { return nil, nil, err } @@ -150,11 +163,16 @@ func iotCorePipelines(m *manifest.Manifest, return buildPipeline, commitPipeline, nil } -func iotCommitManifest(m *manifest.Manifest, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, - repos []rpmmd.RepoConfig, packageSets map[string]rpmmd.PackageSet, +func iotCommitManifest(m *manifest.Manifest, + workload workload.Workload, + t *imageType, + customizations *blueprint.Customizations, + options distro.ImageOptions, + repos []rpmmd.RepoConfig, + packageSets map[string]rpmmd.PackageSet, rng *rand.Rand) error { - buildPipeline, commitPipeline, err := iotCorePipelines(m, t, customizations, options, repos, packageSets) + buildPipeline, commitPipeline, err := iotCorePipelines(m, workload, t, customizations, options, repos, packageSets) if err != nil { return err } @@ -163,10 +181,15 @@ func iotCommitManifest(m *manifest.Manifest, t *imageType, customizations *bluep return nil } -func iotContainerManifest(m *manifest.Manifest, t *imageType, customizations *blueprint.Customizations, - options distro.ImageOptions, repos []rpmmd.RepoConfig, packageSets map[string]rpmmd.PackageSet, +func iotContainerManifest(m *manifest.Manifest, + workload workload.Workload, + t *imageType, + customizations *blueprint.Customizations, + options distro.ImageOptions, + repos []rpmmd.RepoConfig, + packageSets map[string]rpmmd.PackageSet, rng *rand.Rand) error { - buildPipeline, commitPipeline, err := iotCorePipelines(m, t, customizations, options, repos, packageSets) + buildPipeline, commitPipeline, err := iotCorePipelines(m, workload, t, customizations, options, repos, packageSets) if err != nil { return err } @@ -180,6 +203,7 @@ func iotContainerManifest(m *manifest.Manifest, t *imageType, customizations *bl } func containerManifest(m *manifest.Manifest, + workload workload.Workload, t *imageType, customizations *blueprint.Customizations, options distro.ImageOptions, @@ -188,7 +212,7 @@ func containerManifest(m *manifest.Manifest, rng *rand.Rand) error { buildPipeline := manifest.NewBuildPipeline(m, t.arch.distro.runner, repos) - treePipeline, err := osPipeline(m, buildPipeline, t, repos, packageSets[osPkgsKey], packageSets[blueprintPkgsKey], customizations, options, rng) + treePipeline, err := osPipeline(m, buildPipeline, workload, t, repos, packageSets[osPkgsKey], customizations, options, rng) if err != nil { return err } @@ -199,10 +223,10 @@ func containerManifest(m *manifest.Manifest, func osPipeline(m *manifest.Manifest, buildPipeline *manifest.BuildPipeline, + workload workload.Workload, t *imageType, repos []rpmmd.RepoConfig, osPackageSet rpmmd.PackageSet, - blueprintPackageSet rpmmd.PackageSet, c *blueprint.Customizations, options distro.ImageOptions, rng *rand.Rand) (*manifest.OSPipeline, error) { @@ -210,6 +234,7 @@ func osPipeline(m *manifest.Manifest, imageConfig := t.getDefaultImageConfig() pl := manifest.NewOSPipeline(m, buildPipeline, t.platform, repos) + pl.Workload = workload if t.bootable { var err error @@ -249,8 +274,6 @@ func osPipeline(m *manifest.Manifest, pl.ExtraBasePackages = osPackageSet.Include pl.ExcludeBasePackages = osPackageSet.Exclude pl.ExtraBaseRepos = osPackageSet.Repositories - pl.UserPackages = blueprintPackageSet.Include - pl.UserRepos = blueprintPackageSet.Repositories pl.GPGKeyFiles = imageConfig.GPGKeyFiles pl.ExcludeDocs = imageConfig.ExcludeDocs @@ -262,16 +285,8 @@ func osPipeline(m *manifest.Manifest, pl.Users = c.GetUsers() } - services := &blueprint.ServicesCustomization{ - Enabled: imageConfig.EnabledServices, - Disabled: imageConfig.DisabledServices, - } - if extraServices := c.GetServices(); extraServices != nil { - services.Enabled = append(services.Enabled, extraServices.Enabled...) - services.Disabled = append(services.Disabled, extraServices.Disabled...) - } - pl.EnabledServices = services.Enabled - pl.DisabledServices = services.Disabled + pl.EnabledServices = imageConfig.EnabledServices + pl.DisabledServices = imageConfig.DisabledServices pl.DefaultTarget = imageConfig.DefaultTarget pl.Firewall = c.GetFirewall() diff --git a/internal/manifest/os.go b/internal/manifest/os.go index a236df19a..db02f488e 100644 --- a/internal/manifest/os.go +++ b/internal/manifest/os.go @@ -11,6 +11,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/osbuild2" "github.com/osbuild/osbuild-composer/internal/platform" "github.com/osbuild/osbuild-composer/internal/rpmmd" + "github.com/osbuild/osbuild-composer/internal/workload" ) type OSPipelineOSTree struct { @@ -36,11 +37,8 @@ type OSPipeline struct { ExcludeBasePackages []string // Additional repos to install the base packages from. ExtraBaseRepos []rpmmd.RepoConfig - // Packages to install on top of the base packages in a seconadry dnf - // transaction. - UserPackages []string - // Additional repos to install the user packages from. - UserRepos []rpmmd.RepoConfig + // Workload to install on top of the base system + Workload workload.Workload // OSTree configuration, if nil the tree cannot be in an OSTree commit OSTree *OSPipelineOSTree // Partition table, if nil the tree cannot be put on a partioned disk @@ -194,10 +192,10 @@ func (p *OSPipeline) getPackageSetChain() []rpmmd.PackageSet { }, } - if len(p.UserPackages) > 0 { + if p.Workload != nil { chain = append(chain, rpmmd.PackageSet{ - Include: p.UserPackages, - Repositories: append(p.repos, p.UserRepos...), + Include: p.Workload.GetPackages(), + Repositories: append(p.repos, p.Workload.GetRepos()...), }) } @@ -312,11 +310,17 @@ func (p *OSPipeline) serialize() osbuild2.Pipeline { } } - if p.EnabledServices != nil || - p.DisabledServices != nil || p.DefaultTarget != "" { + enabledServices := []string{} + enabledServices = append(enabledServices, p.EnabledServices...) + enabledServices = append(enabledServices, p.Workload.GetServices()...) + disabledServices := []string{} + disabledServices = append(disabledServices, p.DisabledServices...) + disabledServices = append(disabledServices, p.Workload.GetDisabledServices()...) + if len(enabledServices) != 0 || + len(disabledServices) != 0 || p.DefaultTarget != "" { pipeline.AddStage(osbuild2.NewSystemdStage(&osbuild2.SystemdStageOptions{ - EnabledServices: p.EnabledServices, - DisabledServices: p.DisabledServices, + EnabledServices: enabledServices, + DisabledServices: disabledServices, DefaultTarget: p.DefaultTarget, })) } diff --git a/internal/workload/anaconda.go b/internal/workload/anaconda.go new file mode 100644 index 000000000..b4a5e50c0 --- /dev/null +++ b/internal/workload/anaconda.go @@ -0,0 +1,7 @@ +package workload + +// TODO: replace the Anaconda pipeline by the OS pipeline with the +// anaconda workload. +type Anaconda struct { + BaseWorkload +} diff --git a/internal/workload/custom.go b/internal/workload/custom.go new file mode 100644 index 000000000..47f5d3ee3 --- /dev/null +++ b/internal/workload/custom.go @@ -0,0 +1,22 @@ +package workload + +type Custom struct { + BaseWorkload + Packages []string + Services []string + DisabledServices []string +} + +func (p *Custom) GetPackages() []string { + return p.Packages +} + +func (p *Custom) GetServices() []string { + return p.Services +} + +// TODO: Does this belong here? What kind of workload requires +// services to be disabled? +func (p *Custom) GetDisabledServices() []string { + return p.DisabledServices +} diff --git a/internal/workload/sap.go b/internal/workload/sap.go new file mode 100644 index 000000000..0f066b78c --- /dev/null +++ b/internal/workload/sap.go @@ -0,0 +1,6 @@ +package workload + +// TODO! +type SAP struct { + BaseWorkload +} diff --git a/internal/workload/static_webserver.go b/internal/workload/static_webserver.go new file mode 100644 index 000000000..c06924050 --- /dev/null +++ b/internal/workload/static_webserver.go @@ -0,0 +1,7 @@ +package workload + +// TODO: replace the CommitServerTree pipeline by the OS pipeline with the +// StaticWebserver workload. +type StaticWebserver struct { + BaseWorkload +} diff --git a/internal/workload/workload.go b/internal/workload/workload.go new file mode 100644 index 000000000..b147e3f77 --- /dev/null +++ b/internal/workload/workload.go @@ -0,0 +1,30 @@ +package workload + +import "github.com/osbuild/osbuild-composer/internal/rpmmd" + +type Workload interface { + GetPackages() []string + GetRepos() []rpmmd.RepoConfig + GetServices() []string + GetDisabledServices() []string +} + +type BaseWorkload struct { + Repos []rpmmd.RepoConfig +} + +func (p BaseWorkload) GetPackages() []string { + return []string{} +} + +func (p BaseWorkload) GetRepos() []rpmmd.RepoConfig { + return p.Repos +} + +func (p BaseWorkload) GetServices() []string { + return []string{} +} + +func (p BaseWorkload) GetDisabledServices() []string { + return []string{} +}