particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pipeline: clean up and document

Browse source 
This finishes the initial version of the pipline package, adding
documentation, but still missing unittests.

Signed-off-by: Tom Gundersen <teg@jklm.no>
This commit is contained in:
Tom Gundersen 2019-10-03 12:28:25 +02:00
parent 41c6f5dd0b
commit 859bc0ad2e
11 changed files with 230 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

51
internal/pipeline/assembler.go Normal file
View file

@ -0,0 +1,51 @@
package pipeline
import (
"encoding/json"
"errors"
)
// An Assembler turns a filesystem tree into a target image.
type Assembler struct {
Name string `json:"name"`
Options AssemblerOptions `json:"options"`
}
// AssemblerOptions specify the operations of a given assembler-type.
type AssemblerOptions interface {
isAssemblerOptions()
}
type rawAssembler struct {
Name string `json:"name"`
Options json.RawMessage `json:"options"`
}
// UnmarshalJSON unmarshals JSON into an Assembler object. Each type of
// assembler has a custom unmarshaller for its options, selected based on the
// stage name.
func (assembler *Assembler) UnmarshalJSON(data []byte) error {
var rawAssembler rawAssembler
err := json.Unmarshal(data, &rawAssembler)
if err != nil {
return err
}
var options AssemblerOptions
switch rawAssembler.Name {
case "org.osbuild.tar":
options = new(TarAssemblerOptions)
case "org.osbuild.qcow2":
options = new(QEMUAssemblerOptions)
default:
return errors.New("unexpected assembler name")
}
err = json.Unmarshal(rawAssembler.Options, options)
if err != nil {
return err
}
assembler.Name = rawAssembler.Name
assembler.Options = options
return nil
}

21
internal/pipeline/dnf_stage.go
View file

@ -1,5 +1,12 @@
package pipeline
// The DNFStageOptions describe the operations of the DNF stage.
//
// The DNF stage installs a given set of packages from a given repository,
// as it was at a given point in time. This is meant to ensure that given
// a set of DNF stage options, the output should be reproducible. If the
// metadata of the repository has changed since the stage options were
// first generated, the stage may fail.
type DNFStageOptions struct {
Repositories map[string]*DNFRepository `json:"repos"`
Packages []string `json:"packages"`
@ -9,6 +16,8 @@ type DNFStageOptions struct {
func (DNFStageOptions) isStageOptions() {}
// A DNFRepository describes one repository at a given point in time, as well
// as the GPG key needed to verify its correctness.
type DNFRepository struct {
MetaLink string `json:"metalink,omitempty"`
MirrorList string `json:"mirrorlist,omitempty"`
@ -17,6 +26,8 @@ type DNFRepository struct {
Checksum string `json:"checksum,omitempty"`
}
// NewDNFStageOptions creates a new DNFStageOptions object. It contains its
// mandatory fields, but no repositories.
func NewDNFStageOptions(releaseVersion string, baseArchitecture string) *DNFStageOptions {
return &DNFStageOptions{
Repositories: make(map[string]*DNFRepository),
@ -25,6 +36,7 @@ func NewDNFStageOptions(releaseVersion string, baseArchitecture string) *DNFStag
}
}
// NewDNFStage creates a new DNF stage.
func NewDNFStage(options *DNFStageOptions) *Stage {
return &Stage{
Name: "org.osbuild.dnf",
@ -32,15 +44,20 @@ func NewDNFStage(options *DNFStageOptions) *Stage {
}
}
// AddPackage adds a package to a DNFStageOptions object.
func (options *DNFStageOptions) AddPackage(pkg string) {
options.Packages = append(options.Packages, pkg)
}
// AddRepository adds a repository to a DNFStageOptions object.
func (options *DNFStageOptions) AddRepository(name string, repo *DNFRepository) {
options.Repositories[name] = repo
}
// NewDNFRepository creates a new DNFRepository object. Exactly one of the
// argumnets should not be nil.
func NewDNFRepository(metaLink string, mirrorList string, baseURL string) *DNFRepository {
// TODO: verify that exactly one argument is non-nil
return &DNFRepository{
MetaLink: metaLink,
MirrorList: mirrorList,
@ -48,10 +65,14 @@ func NewDNFRepository(metaLink string, mirrorList string, baseURL string) *DNFRe
}
}
// SetGPGKey sets the GPG key for a repository. This is used to verify the
// packages we install.
func (r *DNFRepository) SetGPGKey(gpgKey string) {
r.GPGKey = gpgKey
}
// SetChecksum sets the metadata checksum of a repository. This is used to
// verify that we only operate on a given version of the repository.
func (r *DNFRepository) SetChecksum(checksum string) {
r.Checksum = checksum
}

11
internal/pipeline/fix_bls_stage.go
View file

@ -1,12 +1,21 @@
package pipeline
// A FixBLSStageOptions struct is empty, as the stage takes no options.
//
// The FixBLSStage fixes the paths in the Boot Loader Specification
// snippets installed into /boot. grub2's kernel install script will
// try to guess the correct path to the kernel and bootloader, and adjust
// the boot loader scripts accordingly. When run under OSBuild this does
// not work correctly, so this stage essentially reverts the "fixup".
type FixBLSStageOptions struct {
}
func (FixBLSStageOptions) isStageOptions() {}
// NewFixBLSStage creates a new FixBLSStage.
func NewFixBLSStage() *Stage {
return &Stage{
Name: "org.osbuild.fix-bls",
Name: "org.osbuild.fix-bls",
Options: &FixBLSStageOptions{},
}
}

9
internal/pipeline/fstab_stage.go
View file

@ -2,12 +2,18 @@ package pipeline
import "github.com/google/uuid"
// The FSTabStageOptions describe the content of the /etc/fstab file.
//
// The structure of the options follows the format of /etc/fstab, except
// that filesystem must be identified by their UUID and ommitted fields
// are set to their defaults (if possible).
type FSTabStageOptions struct {
FileSystems []*FSTabEntry `json:"filesystems"`
}
func (FSTabStageOptions) isStageOptions() {}
// NewFSTabStage creates a now FSTabStage object
func NewFSTabStage(options *FSTabStageOptions) *Stage {
return &Stage{
Name: "org.osbuild.fstab",
@ -15,6 +21,8 @@ func NewFSTabStage(options *FSTabStageOptions) *Stage {
}
}
// An FSTabEntry represents one line in /etc/fstab. With the one exception
// that the the spec field must be represented as an UUID.
type FSTabEntry struct {
UUID uuid.UUID `json:"uuid"`
VFSType string `json:"vfs_type"`
@ -24,6 +32,7 @@ type FSTabEntry struct {
PassNo uint64 `json:"passno,omitempty"`
}
// AddFilesystem adds one entry to and FSTabStageOptions object.
func (options *FSTabStageOptions) AddFilesystem(id uuid.UUID, vfsType string, path string, opts string, freq uint64, passNo uint64) {
options.FileSystems = append(options.FileSystems, &FSTabEntry{
UUID: id,

18
internal/pipeline/grub2_stage.go
View file

@ -2,6 +2,14 @@ package pipeline
import "github.com/google/uuid"
// The GRUB2StageOptions describes the bootloader configuration.
//
// The stage is responsible for installing all bootloader files in
// /boot as well as config files in /etc necessary for regenerating
// the configuration in /boot.
//
// Note that it is the role of an assembler to install any necessary
// bootloaders that are stored in the image outside of any filesystem.
type GRUB2StageOptions struct {
RootFilesystemUUID uuid.UUID `json:"root_fs_uuid"`
BootFilesystemUUID uuid.UUID `json:"boot_fs_uuid,omitempty"`
@ -10,6 +18,8 @@ type GRUB2StageOptions struct {
func (GRUB2StageOptions) isStageOptions() {}
// NewGRUB2StageOptions creates a new GRUB2StageOptions object. It sets the
// mandatory options.
func NewGRUB2StageOptions(rootFilesystemUUID uuid.UUID) *GRUB2StageOptions {
return &GRUB2StageOptions{
RootFilesystemUUID: rootFilesystemUUID,
@ -17,6 +27,7 @@ func NewGRUB2StageOptions(rootFilesystemUUID uuid.UUID) *GRUB2StageOptions {
}
// NewGRUB2Stage creates a new GRUB2 stage object.
func NewGRUB2Stage(options *GRUB2StageOptions) *Stage {
return &Stage{
Name: "org.osbuild.grub2",
@ -24,10 +35,17 @@ func NewGRUB2Stage(options *GRUB2StageOptions) *Stage {
}
}
// SetRootFilesystemUUID sets the UUID of the filesystem containing /.
func (options *GRUB2StageOptions) SetRootFilesystemUUID(u uuid.UUID) {
options.RootFilesystemUUID = u
}
// SetBootFilesystemUUID sets the UUID of the filesystem containing /boot.
func (options *GRUB2StageOptions) SetBootFilesystemUUID(u uuid.UUID) {
options.BootFilesystemUUID = u
}
// SetKernelOptions sets the kernel options that should be passed at boot.
func (options *GRUB2StageOptions) SetKernelOptions(kernelOptions string) {
options.KernelOptions = kernelOptions
}

7
internal/pipeline/locale_stage.go
View file

@ -1,17 +1,24 @@
package pipeline
// The LocaleStageOptions describes the image's locale.
//
// A locale is typically specified as language_[territory], where language
// is specified in ISO 639 and territory in ISO 3166.
type LocaleStageOptions struct {
Language string `json:"language"`
}
func (LocaleStageOptions) isStageOptions() {}
// NewLocaleStageOptions creates a new locale stage options object, with
// the mandatory fields set.
func NewLocaleStageOptions(language string) *LocaleStageOptions {
return &LocaleStageOptions{
Language: language,
}
}
// NewLocaleStage creates a new Locale Stage object.
func NewLocaleStage(options *LocaleStageOptions) *Stage {
return &Stage{
Name: "org.osbuild.locale",

111
internal/pipeline/pipeline.go
View file

@ -1,111 +1,32 @@
// Package pipeline provides primitives for representing and (un)marshalling
// OSBuild pipelines.
package pipeline
import (
"encoding/json"
"errors"
)
// A Pipeline represents an OSBuild pipeline
type Pipeline struct {
BuildPipeline *Pipeline `json:"build,omitempty"`
Stages []*Stage `json:"stages,omitempty"`
Assembler *Assembler `json:"assembler,omitempty"`
}
type Stage struct {
Name string `json:"name"`
Options StageOptions `json:"options"`
}
type StageOptions interface {
isStageOptions()
}
type rawStage struct {
Name string `json:"name"`
Options json.RawMessage `json:"options"`
}
func (stage *Stage) UnmarshalJSON(data []byte) error {
var rawStage rawStage
err := json.Unmarshal(data, &rawStage)
if err != nil {
return err
}
var options StageOptions
switch rawStage.Name {
case "org.osbuild.dnf":
options = new(DNFStageOptions)
case "org.osbuild.fix-bls":
options = new(FixBLSStageOptions)
case "org.osbuild.FSTab":
options = new(FSTabStageOptions)
case "org.osbuild.grub2":
options = new(GRUB2StageOptions)
case "org.osbuild.locale":
options = new(LocaleStageOptions)
case "org.osbuild.SELinux":
options = new(SELinuxStageOptions)
default:
return errors.New("unexpected stage name")
}
err = json.Unmarshal(rawStage.Options, options)
if err != nil {
return err
}
stage.Name = rawStage.Name
stage.Options = options
return nil
}
type Assembler struct {
Name string `json:"name"`
Options AssemblerOptions `json:"options"`
}
type AssemblerOptions interface {
isAssemblerOptions()
}
type rawAssembler struct {
Name string `json:"name"`
Options json.RawMessage `json:"options"`
}
func (assembler *Assembler) UnmarshalJSON(data []byte) error {
var rawAssembler rawAssembler
err := json.Unmarshal(data, &rawAssembler)
if err != nil {
return err
}
var options AssemblerOptions
switch rawAssembler.Name {
case "org.osbuild.tar":
options = new(TarAssemblerOptions)
case "org.osbuild.qcow2":
options = new(QCOW2AssemblerOptions)
default:
return errors.New("unexpected assembler name")
}
err = json.Unmarshal(rawAssembler.Options, options)
if err != nil {
return err
}
assembler.Name = rawAssembler.Name
assembler.Options = options
return nil
// BuildPipeline describes how to create the build environment for the
// following stages and assembler.
BuildPipeline *Pipeline `json:"build,omitempty"`
// Sequence of stages that produce the filesystem tree, which is the
// payload of the produced image.
Stages []*Stage `json:"stages,omitempty"`
// Assembler that assembles the filesystem tree into the target image.
Assembler *Assembler `json:"assembler,omitempty"`
}
// SetBuildPipeline sets the pipeline for generating the build environment for
// a pipeline.
func (p *Pipeline) SetBuildPipeline(buildPipeline *Pipeline) {
p.BuildPipeline = buildPipeline
}
// AddStage appends a stage to the list of stages of a pipeline. The stages
// will be executed in the order they are appended.
func (p *Pipeline) AddStage(stage *Stage) {
p.Stages = append(p.Stages, stage)
}
// SetAssembler sets the assembler for a pipeline.
func (p *Pipeline) SetAssembler(assembler *Assembler) {
p.Assembler = assembler
}

24
internal/pipeline/qcow2_assembler.go
View file

@ -2,23 +2,37 @@ package pipeline
import "github.com/google/uuid"
type QCOW2AssemblerOptions struct {
// QEMUAssemblerOptions desrcibe how to assemble a tree into an image using qemu.
//
// The assembler creates an image of a the given size, adds a GRUB2 bootloader
// and a DOS partition table to it with the given PTUUID containing one ext4
// root partition with the given filesystem UUID and installs the filesystem
// tree into it. Finally, the image is converted into the target format and
// stored with the given filename.
type QEMUAssemblerOptions struct {
Format string `json:"format"`
Filename string `json:"filename"`
PTUUID string `json:"ptuuid"`
RootFilesystemUUDI uuid.UUID `json:"root_fs_uuid"`
Size uint64 `json:"size"`
}
func (QCOW2AssemblerOptions) isAssemblerOptions() {}
func (QEMUAssemblerOptions) isAssemblerOptions() {}
func NewQCOW2AssemblerOptions(filename string, rootFilesystemUUID uuid.UUID, size uint64) *QCOW2AssemblerOptions {
return &QCOW2AssemblerOptions{
// NewQEMUAssemblerOptions creates a now QEMUAssemblerOptions object, with all the mandatory
// fields set.
func NewQEMUAssemblerOptions(format string, ptUUID string, filename string, rootFilesystemUUID uuid.UUID, size uint64) *QEMUAssemblerOptions {
return &QEMUAssemblerOptions{
Format: format,
PTUUID: ptUUID,
Filename: filename,
RootFilesystemUUDI: rootFilesystemUUID,
Size: size,
}
}
func NewQCOW2Assembler(options *QCOW2AssemblerOptions) *Assembler {
// NewQEMUAssembler creates a new QEMU Assembler object.
func NewQEMUAssembler(options *QEMUAssemblerOptions) *Assembler {
return &Assembler{
Name: "org.osbuild.qcow2",
Options: options,

7
internal/pipeline/selinux_stage.go
View file

@ -1,17 +1,24 @@
package pipeline
// The SELinuxStageOptions describe how to apply selinux labels.
//
// A file contexts configuration file is sepcified that describes
// the filesystem labels to apply to the image.
type SELinuxStageOptions struct {
FileContexts string `json:"file_contexts"`
}
func (SELinuxStageOptions) isStageOptions() {}
// NewSELinuxStageOptions creates a new SELinuxStaeOptions object, with
// the mandatory fields set.
func NewSELinuxStageOptions(fileContexts string) *SELinuxStageOptions {
return &SELinuxStageOptions{
FileContexts: fileContexts,
}
}
// NewSELinuxStage creates a new SELinux Stage object.
func NewSELinuxStage(options *SELinuxStageOptions) *Stage {
return &Stage{
Name: "org.osbuild.selinux",

63
internal/pipeline/stage.go Normal file
View file

@ -0,0 +1,63 @@
package pipeline
import (
"encoding/json"
"errors"
)
// A Stage transforms a filesystem tree.
type Stage struct {
// Well-known name in reverse domain-name notation, uniquely identifying
// the stage type.
Name string `json:"name"`
// Stage-type specific options fully determining the operations of the
// stage.
Options StageOptions `json:"options"`
}
// StageOptions specify the operations of a given stage-type.
type StageOptions interface {
isStageOptions()
}
type rawStage struct {
Name string `json:"name"`
Options json.RawMessage `json:"options"`
}
// UnmarshalJSON unmarshals JSON into a Stage object. Each type of stage has
// a custom unmarshaller for its options, selected based on the stage name.
func (stage *Stage) UnmarshalJSON(data []byte) error {
var rawStage rawStage
err := json.Unmarshal(data, &rawStage)
if err != nil {
return err
}
var options StageOptions
switch rawStage.Name {
case "org.osbuild.dnf":
options = new(DNFStageOptions)
case "org.osbuild.fix-bls":
// TODO: verify that we can unmarshall this also if "options" is omitted
options = new(FixBLSStageOptions)
case "org.osbuild.fstab":
options = new(FSTabStageOptions)
case "org.osbuild.grub2":
options = new(GRUB2StageOptions)
case "org.osbuild.locale":
options = new(LocaleStageOptions)
case "org.osbuild.selinux":
options = new(SELinuxStageOptions)
default:
return errors.New("unexpected stage name")
}
err = json.Unmarshal(rawStage.Options, options)
if err != nil {
return err
}
stage.Name = rawStage.Name
stage.Options = options
return nil
}

9
internal/pipeline/tar_assembler.go
View file

@ -1,5 +1,9 @@
package pipeline
// TarAssemblerOptions desrcibe how to assemble a tree into a tar ball.
//
// The assembler tars and optionally compresses the tree using the provided
// compression type, and stores the output with the given filename.
type TarAssemblerOptions struct {
Filename string `json:"filename"`
Compression string `json:"compression,omitempty"`
@ -7,12 +11,15 @@ type TarAssemblerOptions struct {
func (TarAssemblerOptions) isAssemblerOptions() {}
// NewTarAssemblerOptions creates a new TarAssemblerOptions object, with the
// mandatory options set.
func NewTarAssemblerOptions(filename string) *TarAssemblerOptions {
return &TarAssemblerOptions{
Filename: filename,
}
}
// NewTarAssembler creates a new Tar Assembler object.
func NewTarAssembler(options *TarAssemblerOptions) *Assembler {
return &Assembler{
Name: "org.osbuild.tar",
@ -20,6 +27,8 @@ func NewTarAssembler(options *TarAssemblerOptions) *Assembler {
}
}
// SetCompression sets the compression type for a given TarAssemblerOptions
// object.
func (options *TarAssemblerOptions) SetCompression(compression string) {
options.Compression = compression
}