From 529bc803dbaf7db2df6b5245c2ff7c906d1316c0 Mon Sep 17 00:00:00 2001 From: Tom Gundersen Date: Fri, 8 Jul 2022 01:34:38 +0100 Subject: [PATCH] runner: introduce runner abstraction For now all it does is represent the name of the runner and what requirements it has of the build pipeline. Move some package definitions from the runner package set to where it belongs. --- cmd/osbuild-playground/main.go | 3 ++- cmd/osbuild-playground/my-image.go | 3 ++- cmd/osbuild-playground/playground.go | 5 +++-- internal/distro/fedora/distro.go | 9 +++++---- internal/manifest/anaconda.go | 2 +- internal/manifest/build.go | 13 ++++++++----- internal/manifest/commit.go | 2 +- internal/manifest/commit_server_tree.go | 2 +- internal/manifest/iso.go | 2 +- internal/manifest/iso_tree.go | 2 +- internal/manifest/live.go | 2 +- internal/manifest/oci_container.go | 2 +- internal/manifest/os.go | 2 +- internal/manifest/pipeline.go | 21 ++++++--------------- internal/manifest/qcow2.go | 2 +- internal/manifest/tar.go | 2 +- internal/manifest/vmdk.go | 2 +- internal/manifest/vpc.go | 2 +- internal/runner/fedora.go | 18 ++++++++++++++++++ internal/runner/linux.go | 15 +++++++++++++++ internal/runner/rhel.go | 24 ++++++++++++++++++++++++ internal/runner/runner.go | 6 ++++++ 22 files changed, 101 insertions(+), 40 deletions(-) create mode 100644 internal/runner/fedora.go create mode 100644 internal/runner/linux.go create mode 100644 internal/runner/rhel.go create mode 100644 internal/runner/runner.go diff --git a/cmd/osbuild-playground/main.go b/cmd/osbuild-playground/main.go index 319f2fe0d..1173d6912 100644 --- a/cmd/osbuild-playground/main.go +++ b/cmd/osbuild-playground/main.go @@ -14,13 +14,14 @@ import ( "github.com/osbuild/osbuild-composer/internal/distroregistry" "github.com/osbuild/osbuild-composer/internal/manifest" "github.com/osbuild/osbuild-composer/internal/rpmmd" + "github.com/osbuild/osbuild-composer/internal/runner" ) var ImageTypes = make(map[string]ImageType) type ImageType interface { Name() string - InstantiateManifest(m *manifest.Manifest, repos []rpmmd.RepoConfig, runner string) error + InstantiateManifest(m *manifest.Manifest, repos []rpmmd.RepoConfig, runner runner.Runner) error GetExports() []string GetCheckpoints() []string } diff --git a/cmd/osbuild-playground/my-image.go b/cmd/osbuild-playground/my-image.go index bfcd36854..cb0932220 100644 --- a/cmd/osbuild-playground/my-image.go +++ b/cmd/osbuild-playground/my-image.go @@ -4,6 +4,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/manifest" "github.com/osbuild/osbuild-composer/internal/platform" "github.com/osbuild/osbuild-composer/internal/rpmmd" + "github.com/osbuild/osbuild-composer/internal/runner" ) func init() { @@ -35,7 +36,7 @@ func (img *MyImage) Name() string { // Return nil when you are done, or an error if something // went wrong. Your manifest will be streamed to osbuild // for building. -func (img *MyImage) InstantiateManifest(m *manifest.Manifest, repos []rpmmd.RepoConfig, runner string) error { +func (img *MyImage) InstantiateManifest(m *manifest.Manifest, repos []rpmmd.RepoConfig, runner runner.Runner) error { // Let's create a simple OCI container! // configure a build pipeline diff --git a/cmd/osbuild-playground/playground.go b/cmd/osbuild-playground/playground.go index 45d2a9e65..69643678d 100644 --- a/cmd/osbuild-playground/playground.go +++ b/cmd/osbuild-playground/playground.go @@ -10,6 +10,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/manifest" "github.com/osbuild/osbuild-composer/internal/osbuild2" "github.com/osbuild/osbuild-composer/internal/rpmmd" + "github.com/osbuild/osbuild-composer/internal/runner" ) func RunPlayground(img ImageType, d distro.Distro, arch distro.Arch, repos map[string][]rpmmd.RepoConfig, state_dir string) { @@ -22,8 +23,8 @@ func RunPlayground(img ImageType, d distro.Distro, arch distro.Arch, repos map[s manifest := manifest.New() - // TODO: figure out the runner situation - err := img.InstantiateManifest(&manifest, repos[arch.Name()], "org.osbuild.fedora36") + // TODO: query distro for runner + err := img.InstantiateManifest(&manifest, repos[arch.Name()], &runner.Fedora{Version: 36}) if err != nil { panic("InstantiateManifest() failed: " + err.Error()) } diff --git a/internal/distro/fedora/distro.go b/internal/distro/fedora/distro.go index e10de0ac0..be12d7ef1 100644 --- a/internal/distro/fedora/distro.go +++ b/internal/distro/fedora/distro.go @@ -14,6 +14,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/manifest" "github.com/osbuild/osbuild-composer/internal/platform" "github.com/osbuild/osbuild-composer/internal/rpmmd" + "github.com/osbuild/osbuild-composer/internal/runner" "github.com/osbuild/osbuild-composer/internal/workload" ) @@ -289,7 +290,7 @@ type distribution struct { vendor string ostreeRefTmpl string isolabelTmpl string - runner string + runner runner.Runner arches map[string]distro.Arch defaultImageConfig *distro.ImageConfig } @@ -311,7 +312,7 @@ var distroMap = map[string]distribution{ vendor: "fedora", ostreeRefTmpl: "fedora/34/%s/iot", isolabelTmpl: "Fedora-34-BaseOS-%s", - runner: "org.osbuild.fedora34", + runner: &runner.Fedora{Version: 34}, defaultImageConfig: defaultDistroImageConfig, }, fedora35Distribution: { @@ -323,7 +324,7 @@ var distroMap = map[string]distribution{ vendor: "fedora", ostreeRefTmpl: "fedora/35/%s/iot", isolabelTmpl: "Fedora-35-BaseOS-%s", - runner: "org.osbuild.fedora35", + runner: &runner.Fedora{Version: 35}, defaultImageConfig: defaultDistroImageConfig, }, fedora36Distribution: { @@ -335,7 +336,7 @@ var distroMap = map[string]distribution{ vendor: "fedora", ostreeRefTmpl: "fedora/36/%s/iot", isolabelTmpl: "Fedora-36-BaseOS-%s", - runner: "org.osbuild.fedora36", + runner: &runner.Fedora{Version: 36}, defaultImageConfig: defaultDistroImageConfig, }, } diff --git a/internal/manifest/anaconda.go b/internal/manifest/anaconda.go index 2684fb72e..2278612a9 100644 --- a/internal/manifest/anaconda.go +++ b/internal/manifest/anaconda.go @@ -52,7 +52,7 @@ func NewAnacondaPipeline(m *Manifest, product, version string) *AnacondaPipeline { p := &AnacondaPipeline{ - BasePipeline: NewBasePipeline(m, "anaconda-tree", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "anaconda-tree", buildPipeline), repos: repos, kernelName: kernelName, arch: arch, diff --git a/internal/manifest/build.go b/internal/manifest/build.go index 0d1fb2a05..3a9e9ea88 100644 --- a/internal/manifest/build.go +++ b/internal/manifest/build.go @@ -3,6 +3,7 @@ package manifest import ( "github.com/osbuild/osbuild-composer/internal/osbuild2" "github.com/osbuild/osbuild-composer/internal/rpmmd" + "github.com/osbuild/osbuild-composer/internal/runner" ) // A BuildPipeline represents the build environment for other pipelines. As a @@ -15,6 +16,7 @@ import ( type BuildPipeline struct { BasePipeline + runner runner.Runner dependents []Pipeline repos []rpmmd.RepoConfig packageSpecs []rpmmd.PackageSpec @@ -22,9 +24,10 @@ type BuildPipeline struct { // NewBuildPipeline creates a new build pipeline from the repositories in repos // and the specified packages. -func NewBuildPipeline(m *Manifest, runner string, repos []rpmmd.RepoConfig) *BuildPipeline { +func NewBuildPipeline(m *Manifest, runner runner.Runner, repos []rpmmd.RepoConfig) *BuildPipeline { pipeline := &BuildPipeline{ - BasePipeline: NewBasePipeline(m, "build", nil, &runner), + BasePipeline: NewBasePipeline(m, "build", nil), + runner: runner, dependents: make([]Pipeline, 0), repos: repos, } @@ -37,17 +40,16 @@ func (p *BuildPipeline) addDependent(dep Pipeline) { } func (p *BuildPipeline) getPackageSetChain() []rpmmd.PackageSet { - // TODO: have a runner abstraction that provides the necessary packages // TODO: make the /usr/bin/cp dependency conditional // TODO: make the /usr/bin/xz dependency conditional packages := []string{ "selinux-policy-targeted", // needed to build the build pipeline "coreutils", // /usr/bin/cp - used all over - "glibc", // ldconfig - used in the runner - "systemd", // systemd-tmpfiles and systemd-sysusers - used in the runner "xz", // usage unclear } + packages = append(packages, p.runner.GetBuildPackages()...) + for _, pipeline := range p.dependents { packages = append(packages, pipeline.getBuildPackages()...) } @@ -83,6 +85,7 @@ func (p *BuildPipeline) serialize() osbuild2.Pipeline { panic("serialization not started") } pipeline := p.BasePipeline.serialize() + pipeline.Runner = p.runner.String() pipeline.AddStage(osbuild2.NewRPMStage(osbuild2.NewRPMStageOptions(p.repos), osbuild2.NewRpmStageSourceFilesInputs(p.packageSpecs))) pipeline.AddStage(osbuild2.NewSELinuxStage(&osbuild2.SELinuxStageOptions{ diff --git a/internal/manifest/commit.go b/internal/manifest/commit.go index e22040327..35bae2d5a 100644 --- a/internal/manifest/commit.go +++ b/internal/manifest/commit.go @@ -21,7 +21,7 @@ func NewOSTreeCommitPipeline(m *Manifest, treePipeline *OSPipeline, ref string) *OSTreeCommitPipeline { p := &OSTreeCommitPipeline{ - BasePipeline: NewBasePipeline(m, "ostree-commit", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "ostree-commit", buildPipeline), treePipeline: treePipeline, ref: ref, } diff --git a/internal/manifest/commit_server_tree.go b/internal/manifest/commit_server_tree.go index 9e6778f6f..dbd8df631 100644 --- a/internal/manifest/commit_server_tree.go +++ b/internal/manifest/commit_server_tree.go @@ -39,7 +39,7 @@ func NewOSTreeCommitServerTreePipeline(m *Manifest, nginxConfigPath, listenPort string) *OSTreeCommitServerTreePipeline { p := &OSTreeCommitServerTreePipeline{ - BasePipeline: NewBasePipeline(m, "container-tree", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "container-tree", buildPipeline), repos: repos, commitPipeline: commitPipeline, nginxConfigPath: nginxConfigPath, diff --git a/internal/manifest/iso.go b/internal/manifest/iso.go index ffb1aaa63..fe6b75e58 100644 --- a/internal/manifest/iso.go +++ b/internal/manifest/iso.go @@ -19,7 +19,7 @@ func NewISOPipeline(m *Manifest, treePipeline *ISOTreePipeline, filename string) *ISOPipeline { p := &ISOPipeline{ - BasePipeline: NewBasePipeline(m, "bootiso", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "bootiso", buildPipeline), treePipeline: treePipeline, filename: filename, } diff --git a/internal/manifest/iso_tree.go b/internal/manifest/iso_tree.go index 8e93437a8..4cb06735a 100644 --- a/internal/manifest/iso_tree.go +++ b/internal/manifest/iso_tree.go @@ -39,7 +39,7 @@ func NewISOTreePipeline(m *Manifest, isoLabel := fmt.Sprintf(isoLabelTmpl, anacondaPipeline.arch) p := &ISOTreePipeline{ - BasePipeline: NewBasePipeline(m, "bootiso-tree", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "bootiso-tree", buildPipeline), anacondaPipeline: anacondaPipeline, isoLabel: isoLabel, osTreeCommit: osTreeCommit, diff --git a/internal/manifest/live.go b/internal/manifest/live.go index e4975a0f4..e3b1307a5 100644 --- a/internal/manifest/live.go +++ b/internal/manifest/live.go @@ -19,7 +19,7 @@ func NewLiveImgPipeline(m *Manifest, treePipeline *OSPipeline, filename string) *LiveImgPipeline { p := &LiveImgPipeline{ - BasePipeline: NewBasePipeline(m, "image", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "image", buildPipeline), treePipeline: treePipeline, filename: filename, } diff --git a/internal/manifest/oci_container.go b/internal/manifest/oci_container.go index 825c4134e..e0c3433c9 100644 --- a/internal/manifest/oci_container.go +++ b/internal/manifest/oci_container.go @@ -22,7 +22,7 @@ func NewOCIContainerPipeline(m *Manifest, architecture, filename string) *OCIContainerPipeline { p := &OCIContainerPipeline{ - BasePipeline: NewBasePipeline(m, "container", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "container", buildPipeline), treePipeline: treePipeline, architecture: architecture, filename: filename, diff --git a/internal/manifest/os.go b/internal/manifest/os.go index ec68af833..d50525281 100644 --- a/internal/manifest/os.go +++ b/internal/manifest/os.go @@ -110,7 +110,7 @@ func NewOSPipeline(m *Manifest, platform platform.Platform, repos []rpmmd.RepoConfig) *OSPipeline { p := &OSPipeline{ - BasePipeline: NewBasePipeline(m, "os", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "os", buildPipeline), repos: repos, platform: platform, Language: "C.UTF-8", diff --git a/internal/manifest/pipeline.go b/internal/manifest/pipeline.go index d8b9f35a1..1f802d72c 100644 --- a/internal/manifest/pipeline.go +++ b/internal/manifest/pipeline.go @@ -28,7 +28,6 @@ type Pipeline interface { type BasePipeline struct { manifest *Manifest name string - runner string build *BuildPipeline } @@ -67,7 +66,7 @@ func (p BasePipeline) getInline() []string { // the build host's filesystem is used as the build root. The runner specifies how to use this // pipeline as a build pipeline, by naming the distro it contains. When the host system is used // as a build root, then the necessary runner is autodetected. -func NewBasePipeline(m *Manifest, name string, build *BuildPipeline, runner *string) BasePipeline { +func NewBasePipeline(m *Manifest, name string, build *BuildPipeline) BasePipeline { p := BasePipeline{ manifest: m, name: name, @@ -77,12 +76,6 @@ func NewBasePipeline(m *Manifest, name string, build *BuildPipeline, runner *str if build.BasePipeline.manifest != m { panic("build pipeline from a different manifest") } - if build.BasePipeline.runner == "" { - panic("build pipeline does not have runner") - } - } - if runner != nil { - p.runner = *runner } return p } @@ -101,13 +94,11 @@ func (p BasePipeline) serializeEnd() { // meant to be treated as opaque and not to be modified further outside of the pipeline // package. func (p BasePipeline) serialize() osbuild2.Pipeline { - var buildName string + pipeline := osbuild2.Pipeline{ + Name: p.name, + } if p.build != nil { - buildName = "name:" + p.build.Name() - } - return osbuild2.Pipeline{ - Name: p.name, - Runner: p.runner, - Build: buildName, + pipeline.Build = "name:" + p.build.Name() } + return pipeline } diff --git a/internal/manifest/qcow2.go b/internal/manifest/qcow2.go index a3903a595..f04622c77 100644 --- a/internal/manifest/qcow2.go +++ b/internal/manifest/qcow2.go @@ -21,7 +21,7 @@ func NewQCOW2Pipeline(m *Manifest, imgPipeline *LiveImgPipeline, filename string) *QCOW2Pipeline { p := &QCOW2Pipeline{ - BasePipeline: NewBasePipeline(m, "qcow2", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "qcow2", buildPipeline), imgPipeline: imgPipeline, filename: filename, } diff --git a/internal/manifest/tar.go b/internal/manifest/tar.go index c175ced69..7f55fdda4 100644 --- a/internal/manifest/tar.go +++ b/internal/manifest/tar.go @@ -20,7 +20,7 @@ func NewTarPipeline(m *Manifest, pipelinename, filename string) *TarPipeline { p := &TarPipeline{ - BasePipeline: NewBasePipeline(m, pipelinename, buildPipeline, nil), + BasePipeline: NewBasePipeline(m, pipelinename, buildPipeline), inputPipeline: inputPipeline, filename: filename, } diff --git a/internal/manifest/vmdk.go b/internal/manifest/vmdk.go index 756e24946..8bb58d441 100644 --- a/internal/manifest/vmdk.go +++ b/internal/manifest/vmdk.go @@ -19,7 +19,7 @@ func NewVMDKPipeline(m *Manifest, imgPipeline *LiveImgPipeline, filename string) *VMDKPipeline { p := &VMDKPipeline{ - BasePipeline: NewBasePipeline(m, "vmdk", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "vmdk", buildPipeline), imgPipeline: imgPipeline, filename: filename, } diff --git a/internal/manifest/vpc.go b/internal/manifest/vpc.go index 35df0ffd0..ca694c95d 100644 --- a/internal/manifest/vpc.go +++ b/internal/manifest/vpc.go @@ -20,7 +20,7 @@ func NewVPCPipeline(m *Manifest, imgPipeline *LiveImgPipeline, filename string) *VPCPipeline { p := &VPCPipeline{ - BasePipeline: NewBasePipeline(m, "vpc", buildPipeline, nil), + BasePipeline: NewBasePipeline(m, "vpc", buildPipeline), imgPipeline: imgPipeline, filename: filename, } diff --git a/internal/runner/fedora.go b/internal/runner/fedora.go new file mode 100644 index 000000000..cf555bab8 --- /dev/null +++ b/internal/runner/fedora.go @@ -0,0 +1,18 @@ +package runner + +import "fmt" + +type Fedora struct { + Version uint64 +} + +func (r *Fedora) String() string { + return fmt.Sprintf("org.osbuild.fedora%d", r.Version) +} + +func (p *Fedora) GetBuildPackages() []string { + return []string{ + "glibc", // ldconfig + "systemd", // systemd-tmpfiles and systemd-sysusers + } +} diff --git a/internal/runner/linux.go b/internal/runner/linux.go new file mode 100644 index 000000000..5f42a843d --- /dev/null +++ b/internal/runner/linux.go @@ -0,0 +1,15 @@ +package runner + +type Linux struct { +} + +func (r *Linux) String() string { + return "org.osbuild.linux" +} + +func (p *Linux) GetBuildPackages() []string { + return []string{ + "glibc", // ldconfig + "systemd", // systemd-tmpfiles and systemd-sysusers + } +} diff --git a/internal/runner/rhel.go b/internal/runner/rhel.go new file mode 100644 index 000000000..dd17e4f61 --- /dev/null +++ b/internal/runner/rhel.go @@ -0,0 +1,24 @@ +package runner + +import "fmt" + +type RHEL struct { + Major uint64 + Minor uint64 +} + +func (r *RHEL) String() string { + return fmt.Sprintf("org.osbuild.fedora%d%d", r.Major, r.Minor) +} + +func (p *RHEL) GetBuildPackages() []string { + packages := []string{ + "glibc", // ldconfig + } + if p.Major >= 8 { + packages = append(packages, + "systemd", // systemd-tmpfiles and systemd-sysusers + ) + } + return packages +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go new file mode 100644 index 000000000..60679ba28 --- /dev/null +++ b/internal/runner/runner.go @@ -0,0 +1,6 @@ +package runner + +type Runner interface { + String() string + GetBuildPackages() []string +}