particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pipeline: further cleanups

Browse source 
This adds more documentation and makes more properties implicitly inherited rather than
repeated. This makes for less boilerplate, and gives us fewer things to keep in sync.
This commit is contained in:
Tom Gundersen 2022-06-26 20:01:09 +01:00
parent e5ef7933ac
commit c001af63ec
8 changed files with 121 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

59
internal/distro/fedora/pipelines.go
View file

@ -23,11 +23,10 @@ func qcow2Pipelines(t *imageType, customizations *blueprint.Customizations, opti
}
pipelines = append(pipelines, treePipeline.Serialize())
diskfile := "disk.img"
imagePipeline := liveImagePipeline(&buildPipeline, &treePipeline, diskfile, t.arch)
imagePipeline := pipeline.NewLiveImgPipeline(&buildPipeline, &treePipeline, "disk.img")
pipelines = append(pipelines, imagePipeline.Serialize())
qemuPipeline := qemuPipeline(&buildPipeline, &imagePipeline, diskfile, t.filename, osbuild.QEMUFormatQCOW2, osbuild.QCOW2Options{Compat: "1.1"})
qemuPipeline := qemuPipeline(&buildPipeline, &imagePipeline, t.filename, osbuild.QEMUFormatQCOW2, osbuild.QCOW2Options{Compat: "1.1"})
pipelines = append(pipelines, qemuPipeline.Serialize())
return pipelines, nil
@ -45,11 +44,10 @@ func vhdPipelines(t *imageType, customizations *blueprint.Customizations, option
}
pipelines = append(pipelines, treePipeline.Serialize())
diskfile := "disk.img"
imagePipeline := liveImagePipeline(&buildPipeline, &treePipeline, diskfile, t.arch)
imagePipeline := pipeline.NewLiveImgPipeline(&buildPipeline, &treePipeline, "disk.img")
pipelines = append(pipelines, imagePipeline.Serialize())
qemuPipeline := qemuPipeline(&buildPipeline, &imagePipeline, diskfile, t.filename, osbuild.QEMUFormatVPC, nil)
qemuPipeline := qemuPipeline(&buildPipeline, &imagePipeline, t.filename, osbuild.QEMUFormatVPC, nil)
pipelines = append(pipelines, qemuPipeline.Serialize())
return pipelines, nil
}
@ -66,11 +64,10 @@ func vmdkPipelines(t *imageType, customizations *blueprint.Customizations, optio
}
pipelines = append(pipelines, treePipeline.Serialize())
diskfile := "disk.img"
imagePipeline := liveImagePipeline(&buildPipeline, &treePipeline, diskfile, t.arch)
imagePipeline := pipeline.NewLiveImgPipeline(&buildPipeline, &treePipeline, "disk.img")
pipelines = append(pipelines, imagePipeline.Serialize())
qemuPipeline := qemuPipeline(&buildPipeline, &imagePipeline, diskfile, t.filename, osbuild.QEMUFormatVMDK, osbuild.VMDKOptions{Subformat: osbuild.VMDKSubformatStreamOptimized})
qemuPipeline := qemuPipeline(&buildPipeline, &imagePipeline, t.filename, osbuild.QEMUFormatVMDK, osbuild.VMDKOptions{Subformat: osbuild.VMDKSubformatStreamOptimized})
pipelines = append(pipelines, qemuPipeline.Serialize())
return pipelines, nil
}
@ -87,11 +84,10 @@ func openstackPipelines(t *imageType, customizations *blueprint.Customizations,
}
pipelines = append(pipelines, treePipeline.Serialize())
diskfile := "disk.img"
imagePipeline := liveImagePipeline(&buildPipeline, &treePipeline, diskfile, t.arch)
imagePipeline := pipeline.NewLiveImgPipeline(&buildPipeline, &treePipeline, "disk.img")
pipelines = append(pipelines, imagePipeline.Serialize())
qemuPipeline := qemuPipeline(&buildPipeline, &imagePipeline, diskfile, t.filename, osbuild.QEMUFormatQCOW2, nil)
qemuPipeline := qemuPipeline(&buildPipeline, &imagePipeline, t.filename, osbuild.QEMUFormatQCOW2, nil)
pipelines = append(pipelines, qemuPipeline.Serialize())
return pipelines, nil
}
@ -110,7 +106,7 @@ func ec2CommonPipelines(t *imageType, customizations *blueprint.Customizations,
}
pipelines = append(pipelines, treePipeline.Serialize())
imagePipeline := liveImagePipeline(&buildPipeline, &treePipeline, diskfile, t.arch)
imagePipeline := pipeline.NewLiveImgPipeline(&buildPipeline, &treePipeline, diskfile)
pipelines = append(pipelines, imagePipeline.Serialize())
return pipelines, nil
}
@ -155,8 +151,7 @@ func iotCommitPipelines(t *imageType, customizations *blueprint.Customizations,
if err != nil {
return nil, err
}
tarPipeline := pipeline.NewTarPipeline(buildPipeline, &commitPipeline.Pipeline, "commit-archive")
tarPipeline.Filename = t.Filename()
tarPipeline := pipeline.NewTarPipeline(buildPipeline, &commitPipeline.Pipeline, "commit-archive", t.Filename())
pipelines = append(pipelines, buildPipeline.Serialize(), treePipeline.Serialize(), commitPipeline.Serialize(), tarPipeline.Serialize())
return pipelines, nil
}
@ -198,16 +193,16 @@ func osPipeline(buildPipeline *pipeline.BuildPipeline,
}
}
pl := pipeline.NewOSPipeline(buildPipeline, t.rpmOstree, repos, packages, pt, c.GetKernel().Name)
var bootLoader pipeline.BootLoader
if t.Arch().Name() == distro.S390xArchName {
pl.BootLoader = pipeline.BOOTLOADER_ZIPL
bootLoader = pipeline.BOOTLOADER_ZIPL
} else {
pl.BootLoader = pipeline.BOOTLOADER_GRUB
bootLoader = pipeline.BOOTLOADER_GRUB
}
pl := pipeline.NewOSPipeline(buildPipeline, t.rpmOstree, options.OSTree.Parent, repos, packages, pt, bootLoader, t.arch.legacy, c.GetKernel().Name)
pl.UEFI = t.supportsUEFI()
pl.GRUBLegacy = t.arch.legacy
pl.Vendor = t.arch.distro.vendor
var kernelOptions []string
@ -219,8 +214,6 @@ func osPipeline(buildPipeline *pipeline.BuildPipeline,
}
pl.KernelOptionsAppend = kernelOptions
pl.OSTreeParent = options.OSTree.Parent
pl.OSTreeURL = options.OSTree.URL
pl.GPGKeyFiles = imageConfig.GPGKeyFiles
if !t.bootISO {
@ -298,7 +291,6 @@ func osPipeline(buildPipeline *pipeline.BuildPipeline,
func ostreeCommitPipeline(buildPipeline *pipeline.BuildPipeline, treePipeline *pipeline.OSPipeline, options distro.ImageOptions, osVersion string) pipeline.OSTreeCommitPipeline {
p := pipeline.NewOSTreeCommitPipeline(buildPipeline, treePipeline, options.OSTree.Ref)
p.OSVersion = osVersion
p.Parent = options.OSTree.Parent
return p
}
@ -352,25 +344,8 @@ func bootISOPipeline(buildPipeline *pipeline.BuildPipeline, treePipeline *pipeli
return p
}
func liveImagePipeline(buildPipeline *pipeline.BuildPipeline, treePipeline *pipeline.OSPipeline, outputFilename string, arch *architecture) pipeline.LiveImgPipeline {
p := pipeline.NewLiveImgPipeline(buildPipeline, treePipeline)
p.Filename = outputFilename
if arch.name == distro.S390xArchName {
p.BootLoader = pipeline.BOOTLOADER_ZIPL
} else {
p.BootLoader = pipeline.BOOTLOADER_GRUB
p.GRUBLegacy = arch.legacy
}
return p
}
func qemuPipeline(buildPipeline *pipeline.BuildPipeline, imagePipeline *pipeline.LiveImgPipeline, inputFilename, outputFilename string, format osbuild.QEMUFormat, formatOptions osbuild.QEMUFormatOptions) pipeline.QemuPipeline {
p := pipeline.NewQemuPipeline(buildPipeline, imagePipeline, string(format))
p.InputFilename = inputFilename
p.OutputFilename = outputFilename
func qemuPipeline(buildPipeline *pipeline.BuildPipeline, imagePipeline *pipeline.LiveImgPipeline, outputFilename string, format osbuild.QEMUFormat, formatOptions osbuild.QEMUFormatOptions) pipeline.QemuPipeline {
p := pipeline.NewQemuPipeline(buildPipeline, imagePipeline, string(format), outputFilename)
p.Format = format
p.FormatOptions = formatOptions

3
internal/pipeline/commit.go
View file

@ -8,7 +8,6 @@ type OSTreeCommitPipeline struct {
Pipeline
treePipeline *OSPipeline
OSVersion string
Parent string
ref string
}
@ -39,7 +38,7 @@ func (p OSTreeCommitPipeline) Serialize() osbuild2.Pipeline {
&osbuild2.OSTreeCommitStageOptions{
Ref: p.Ref(),
OSVersion: p.OSVersion,
Parent: p.Parent,
Parent: p.treePipeline.OSTreeParent(),
},
&osbuild2.OSTreeCommitStageInputs{Tree: commitStageInput}),
)

7
internal/pipeline/commit_server_tree.go
View file

@ -10,9 +10,10 @@ import (
type OSTreeCommitServerTreePipeline struct {
Pipeline
commitPipeline *OSTreeCommitPipeline
Repos []rpmmd.RepoConfig
PackageSpecs []rpmmd.PackageSpec
commitPipeline *OSTreeCommitPipeline
Repos []rpmmd.RepoConfig
PackageSpecs []rpmmd.PackageSpec
// TODO: should this be configurable?
Language string
NginxConfigPath string
ListenPort string

26
internal/pipeline/live.go
View file

@ -6,19 +6,23 @@ import (
type LiveImgPipeline struct {
Pipeline
BootLoader BootLoader
GRUBLegacy string
treePipeline *OSPipeline
Filename string
filename string
}
func NewLiveImgPipeline(buildPipeline *BuildPipeline, treePipeline *OSPipeline) LiveImgPipeline {
func NewLiveImgPipeline(buildPipeline *BuildPipeline, treePipeline *OSPipeline, filename string) LiveImgPipeline {
return LiveImgPipeline{
Pipeline: New("image", buildPipeline, nil),
treePipeline: treePipeline,
filename: filename,
}
}
func (p LiveImgPipeline) Filename() string {
return p.filename
}
func (p LiveImgPipeline) Serialize() osbuild2.Pipeline {
pipeline := p.Pipeline.Serialize()
@ -27,26 +31,26 @@ func (p LiveImgPipeline) Serialize() osbuild2.Pipeline {
panic("no partition table in live image")
}
for _, stage := range osbuild2.GenImagePrepareStages(pt, p.Filename, osbuild2.PTSfdisk) {
for _, stage := range osbuild2.GenImagePrepareStages(pt, p.Filename(), osbuild2.PTSfdisk) {
pipeline.AddStage(stage)
}
inputName := "root-tree"
copyOptions, copyDevices, copyMounts := osbuild2.GenCopyFSTreeOptions(inputName, p.treePipeline.Name(), p.Filename, pt)
copyOptions, copyDevices, copyMounts := osbuild2.GenCopyFSTreeOptions(inputName, p.treePipeline.Name(), p.Filename(), pt)
copyInputs := osbuild2.NewCopyStagePipelineTreeInputs(inputName, p.treePipeline.Name())
pipeline.AddStage(osbuild2.NewCopyStage(copyOptions, copyInputs, copyDevices, copyMounts))
for _, stage := range osbuild2.GenImageFinishStages(pt, p.Filename) {
for _, stage := range osbuild2.GenImageFinishStages(pt, p.Filename()) {
pipeline.AddStage(stage)
}
switch p.BootLoader {
switch p.treePipeline.BootLoader() {
case BOOTLOADER_GRUB:
if p.GRUBLegacy != "" {
pipeline.AddStage(osbuild2.NewGrub2InstStage(osbuild2.NewGrub2InstStageOption(p.Filename, pt, p.GRUBLegacy)))
if grubLegacy := p.treePipeline.GRUBLegacy(); grubLegacy != "" {
pipeline.AddStage(osbuild2.NewGrub2InstStage(osbuild2.NewGrub2InstStageOption(p.Filename(), pt, grubLegacy)))
}
case BOOTLOADER_ZIPL:
loopback := osbuild2.NewLoopbackDevice(&osbuild2.LoopbackDeviceOptions{Filename: p.Filename})
loopback := osbuild2.NewLoopbackDevice(&osbuild2.LoopbackDeviceOptions{Filename: p.Filename()})
pipeline.AddStage(osbuild2.NewZiplInstStage(osbuild2.NewZiplInstStageOptions(p.treePipeline.KernelVer(), pt), loopback, copyDevices, copyMounts))
}

83
internal/pipeline/os.go
View file

@ -12,24 +12,33 @@ import (
"github.com/osbuild/osbuild-composer/internal/rpmmd"
)
type BootLoader uint64
const (
BOOTLOADER_GRUB BootLoader = iota
BOOTLOADER_ZIPL
)
// OSPipeline represents the filesystem tree of the target image. This roughly
// correpsonds to the root filesystem once an instance of the image is running.
type OSPipeline struct {
Pipeline
OSTreeParent string
OSTreeURL string
// KernelOptionsAppend are appended to the kernel commandline
KernelOptionsAppend []string
BootLoader BootLoader
UEFI bool
GRUBLegacy string
Vendor string
GPGKeyFiles []string
Language string
Keyboard *string
Hostname string
Timezone string
NTPServers []string
EnabledServices []string
DisabledServices []string
DefaultTarget string
// UEFI indicates whether or not the OS should support UEFI
UEFI bool
Vendor string
// GPGKeyFiles are a list of filenames in the OS which will be imported
// as GPG keys into the RPM database.
GPGKeyFiles []string
Language string
Keyboard *string
Hostname string
Timezone string
NTPServers []string
EnabledServices []string
DisabledServices []string
DefaultTarget string
// TODO: drop blueprint types from the API
Groups []blueprint.GroupCustomization
Users []blueprint.UserCustomization
@ -55,13 +64,30 @@ type OSPipeline struct {
WAAgentConfig *osbuild2.WAAgentConfStageOptions
osTree bool
osTreeParent string
repos []rpmmd.RepoConfig
packageSpecs []rpmmd.PackageSpec
partitionTable *disk.PartitionTable
bootLoader BootLoader
grubLegacy string
kernelVer string
}
func NewOSPipeline(buildPipeline *BuildPipeline, osTree bool, repos []rpmmd.RepoConfig, packages []rpmmd.PackageSpec, pt *disk.PartitionTable, kernelName string) OSPipeline {
// NewOSPipeline creates a new OS pipeline. osTree indicates whether or not the
// system is ostree based. osTreeParent indicates (for ostree systems) what the
// parent commit is. repos are the reposotories to install RPMs from. packages
// are the depsolved pacakges to be installed into the tree. partitionTable
// represents the disk layout of the target system. kernelName is the name of the
// kernel package that will be used on the target system.
func NewOSPipeline(buildPipeline *BuildPipeline,
osTree bool,
osTreeParent string,
repos []rpmmd.RepoConfig,
packages []rpmmd.PackageSpec,
partitionTable *disk.PartitionTable,
bootLoader BootLoader,
grubLegacy string,
kernelName string) OSPipeline {
name := "os"
if osTree {
name = "ostree-tree"
@ -70,14 +96,21 @@ func NewOSPipeline(buildPipeline *BuildPipeline, osTree bool, repos []rpmmd.Repo
return OSPipeline{
Pipeline: New(name, buildPipeline, nil),
osTree: osTree,
osTreeParent: osTreeParent,
repos: repos,
packageSpecs: packages,
partitionTable: pt,
partitionTable: partitionTable,
bootLoader: bootLoader,
grubLegacy: grubLegacy,
kernelVer: kernelVer,
Hostname: "localhost.localdomain",
}
}
func (p OSPipeline) OSTreeParent() string {
return p.osTreeParent
}
func (p OSPipeline) KernelVer() string {
return p.kernelVer
}
@ -86,11 +119,19 @@ func (p OSPipeline) PartitionTable() *disk.PartitionTable {
return p.partitionTable
}
func (p OSPipeline) BootLoader() BootLoader {
return p.bootLoader
}
func (p OSPipeline) GRUBLegacy() string {
return p.grubLegacy
}
func (p OSPipeline) Serialize() osbuild2.Pipeline {
pipeline := p.Pipeline.Serialize()
if p.osTree && p.OSTreeParent != "" && p.OSTreeURL != "" {
pipeline.AddStage(osbuild2.NewOSTreePasswdStage("org.osbuild.source", p.OSTreeParent))
if p.osTree && p.OSTreeParent() != "" {
pipeline.AddStage(osbuild2.NewOSTreePasswdStage("org.osbuild.source", p.OSTreeParent()))
}
rpmOptions := osbuild2.NewRPMStageOptions(p.repos)
@ -241,9 +282,9 @@ func (p OSPipeline) Serialize() osbuild2.Pipeline {
pipeline.AddStage(osbuild2.NewFSTabStage(osbuild2.NewFSTabStageOptions(pt)))
var bootloader *osbuild2.Stage
switch p.BootLoader {
switch p.BootLoader() {
case BOOTLOADER_GRUB:
options := osbuild2.NewGrub2StageOptionsUnified(pt, p.kernelVer, p.UEFI, p.GRUBLegacy, p.Vendor, false)
options := osbuild2.NewGrub2StageOptionsUnified(pt, p.kernelVer, p.UEFI, p.GRUBLegacy(), p.Vendor, false)
if cfg := p.Grub2Config; cfg != nil {
// TODO: don't store Grub2Config in OSPipeline, making the overrides unnecessary
// grub2.Config.Default is owned and set by `NewGrub2StageOptionsUnified`

7
internal/pipeline/pipeline.go
View file

@ -10,13 +10,6 @@ import (
"github.com/osbuild/osbuild-composer/internal/osbuild2"
)
type BootLoader uint64
const (
BOOTLOADER_GRUB BootLoader = iota
BOOTLOADER_ZIPL
)
// A Pipeline represents the core functionality shared between each of the pipeline
// implementations, and the Pipeline struct must be embedded in each of them.
type Pipeline struct {

21
internal/pipeline/qemu.go
View file

@ -4,20 +4,25 @@ import (
"github.com/osbuild/osbuild-composer/internal/osbuild2"
)
// A QeumPipeline turns a raw image file into qemu-based image format, such as qcow2.
type QemuPipeline struct {
Pipeline
imgPipeline *Pipeline
InputFilename string
OutputFilename string
// TODO: don't expose the osbuild2 types in the API
Format osbuild2.QEMUFormat
FormatOptions osbuild2.QEMUFormatOptions
imgPipeline *LiveImgPipeline
filename string
}
func NewQemuPipeline(buildPipeline *BuildPipeline, imgPipeline *LiveImgPipeline, name string) QemuPipeline {
// NewQemuPipeline createsa new Qemu pipeline. imgPipeline is the pipeline producing the
// raw image. The pipeline name is the name of the new pipeline. Filename is the name
// of the produced image.
func NewQemuPipeline(buildPipeline *BuildPipeline, imgPipeline *LiveImgPipeline, pipelinename, filename string) QemuPipeline {
return QemuPipeline{
Pipeline: New(name, buildPipeline, nil),
imgPipeline: &imgPipeline.Pipeline,
Pipeline: New(pipelinename, buildPipeline, nil),
imgPipeline: imgPipeline,
filename: filename,
}
}
@ -25,8 +30,8 @@ func (p QemuPipeline) Serialize() osbuild2.Pipeline {
pipeline := p.Pipeline.Serialize()
pipeline.AddStage(osbuild2.NewQEMUStage(
osbuild2.NewQEMUStageOptions(p.OutputFilename, p.Format, p.FormatOptions),
osbuild2.NewQemuStagePipelineFilesInputs(p.imgPipeline.Name(), p.InputFilename),
osbuild2.NewQEMUStageOptions(p.filename, p.Format, p.FormatOptions),
osbuild2.NewQemuStagePipelineFilesInputs(p.imgPipeline.Name(), p.imgPipeline.Filename()),
))
return pipeline

13
internal/pipeline/tar.go
View file

@ -4,16 +4,21 @@ import (
"github.com/osbuild/osbuild-composer/internal/osbuild2"
)
// A TarPipeline represents the contents of another pipeline in a tar file
type TarPipeline struct {
Pipeline
inputPipeline *Pipeline
Filename string
filename string
}
func NewTarPipeline(buildPipeline *BuildPipeline, inputPipeline *Pipeline, name string) TarPipeline {
// NewTarPipeline creates a new TarPipeline. The inputPipeline represents the
// filesystem tree which will be the contents of the tar file. The pipelinename
// is the name of the pipeline. The filename is the name of the output tar file.
func NewTarPipeline(buildPipeline *BuildPipeline, inputPipeline *Pipeline, pipelinename, filename string) TarPipeline {
return TarPipeline{
Pipeline: New(name, buildPipeline, nil),
Pipeline: New(pipelinename, buildPipeline, nil),
inputPipeline: inputPipeline,
filename: filename,
}
}
@ -24,7 +29,7 @@ func (p TarPipeline) Serialize() osbuild2.Pipeline {
tree.Type = "org.osbuild.tree"
tree.Origin = "org.osbuild.pipeline"
tree.References = []string{"name:" + p.inputPipeline.Name()}
tarStage := osbuild2.NewTarStage(&osbuild2.TarStageOptions{Filename: p.Filename}, &osbuild2.TarStageInputs{Tree: tree})
tarStage := osbuild2.NewTarStage(&osbuild2.TarStageOptions{Filename: p.filename}, &osbuild2.TarStageInputs{Tree: tree})
pipeline.AddStage(tarStage)