diff --git a/internal/pipeline/assembler.go b/internal/pipeline/assembler.go new file mode 100644 index 000000000..558e3d7e5 --- /dev/null +++ b/internal/pipeline/assembler.go @@ -0,0 +1,51 @@ +package pipeline + +import ( + "encoding/json" + "errors" +) + +// An Assembler turns a filesystem tree into a target image. +type Assembler struct { + Name string `json:"name"` + Options AssemblerOptions `json:"options"` +} + +// AssemblerOptions specify the operations of a given assembler-type. +type AssemblerOptions interface { + isAssemblerOptions() +} + +type rawAssembler struct { + Name string `json:"name"` + Options json.RawMessage `json:"options"` +} + +// UnmarshalJSON unmarshals JSON into an Assembler object. Each type of +// assembler has a custom unmarshaller for its options, selected based on the +// stage name. +func (assembler *Assembler) UnmarshalJSON(data []byte) error { + var rawAssembler rawAssembler + err := json.Unmarshal(data, &rawAssembler) + if err != nil { + return err + } + var options AssemblerOptions + switch rawAssembler.Name { + case "org.osbuild.tar": + options = new(TarAssemblerOptions) + case "org.osbuild.qcow2": + options = new(QEMUAssemblerOptions) + default: + return errors.New("unexpected assembler name") + } + err = json.Unmarshal(rawAssembler.Options, options) + if err != nil { + return err + } + + assembler.Name = rawAssembler.Name + assembler.Options = options + + return nil +} diff --git a/internal/pipeline/dnf_stage.go b/internal/pipeline/dnf_stage.go index eb9743959..2b06b1039 100644 --- a/internal/pipeline/dnf_stage.go +++ b/internal/pipeline/dnf_stage.go @@ -1,5 +1,12 @@ package pipeline +// The DNFStageOptions describe the operations of the DNF stage. +// +// The DNF stage installs a given set of packages from a given repository, +// as it was at a given point in time. This is meant to ensure that given +// a set of DNF stage options, the output should be reproducible. If the +// metadata of the repository has changed since the stage options were +// first generated, the stage may fail. type DNFStageOptions struct { Repositories map[string]*DNFRepository `json:"repos"` Packages []string `json:"packages"` @@ -9,6 +16,8 @@ type DNFStageOptions struct { func (DNFStageOptions) isStageOptions() {} +// A DNFRepository describes one repository at a given point in time, as well +// as the GPG key needed to verify its correctness. type DNFRepository struct { MetaLink string `json:"metalink,omitempty"` MirrorList string `json:"mirrorlist,omitempty"` @@ -17,6 +26,8 @@ type DNFRepository struct { Checksum string `json:"checksum,omitempty"` } +// NewDNFStageOptions creates a new DNFStageOptions object. It contains its +// mandatory fields, but no repositories. func NewDNFStageOptions(releaseVersion string, baseArchitecture string) *DNFStageOptions { return &DNFStageOptions{ Repositories: make(map[string]*DNFRepository), @@ -25,6 +36,7 @@ func NewDNFStageOptions(releaseVersion string, baseArchitecture string) *DNFStag } } +// NewDNFStage creates a new DNF stage. func NewDNFStage(options *DNFStageOptions) *Stage { return &Stage{ Name: "org.osbuild.dnf", @@ -32,15 +44,20 @@ func NewDNFStage(options *DNFStageOptions) *Stage { } } +// AddPackage adds a package to a DNFStageOptions object. func (options *DNFStageOptions) AddPackage(pkg string) { options.Packages = append(options.Packages, pkg) } +// AddRepository adds a repository to a DNFStageOptions object. func (options *DNFStageOptions) AddRepository(name string, repo *DNFRepository) { options.Repositories[name] = repo } +// NewDNFRepository creates a new DNFRepository object. Exactly one of the +// argumnets should not be nil. func NewDNFRepository(metaLink string, mirrorList string, baseURL string) *DNFRepository { + // TODO: verify that exactly one argument is non-nil return &DNFRepository{ MetaLink: metaLink, MirrorList: mirrorList, @@ -48,10 +65,14 @@ func NewDNFRepository(metaLink string, mirrorList string, baseURL string) *DNFRe } } +// SetGPGKey sets the GPG key for a repository. This is used to verify the +// packages we install. func (r *DNFRepository) SetGPGKey(gpgKey string) { r.GPGKey = gpgKey } +// SetChecksum sets the metadata checksum of a repository. This is used to +// verify that we only operate on a given version of the repository. func (r *DNFRepository) SetChecksum(checksum string) { r.Checksum = checksum } diff --git a/internal/pipeline/fix_bls_stage.go b/internal/pipeline/fix_bls_stage.go index fc19c1ea3..769514226 100644 --- a/internal/pipeline/fix_bls_stage.go +++ b/internal/pipeline/fix_bls_stage.go @@ -1,12 +1,21 @@ package pipeline +// A FixBLSStageOptions struct is empty, as the stage takes no options. +// +// The FixBLSStage fixes the paths in the Boot Loader Specification +// snippets installed into /boot. grub2's kernel install script will +// try to guess the correct path to the kernel and bootloader, and adjust +// the boot loader scripts accordingly. When run under OSBuild this does +// not work correctly, so this stage essentially reverts the "fixup". type FixBLSStageOptions struct { } func (FixBLSStageOptions) isStageOptions() {} +// NewFixBLSStage creates a new FixBLSStage. func NewFixBLSStage() *Stage { return &Stage{ - Name: "org.osbuild.fix-bls", + Name: "org.osbuild.fix-bls", + Options: &FixBLSStageOptions{}, } } diff --git a/internal/pipeline/fstab_stage.go b/internal/pipeline/fstab_stage.go index c1e367b0c..3a642d4f4 100644 --- a/internal/pipeline/fstab_stage.go +++ b/internal/pipeline/fstab_stage.go @@ -2,12 +2,18 @@ package pipeline import "github.com/google/uuid" +// The FSTabStageOptions describe the content of the /etc/fstab file. +// +// The structure of the options follows the format of /etc/fstab, except +// that filesystem must be identified by their UUID and ommitted fields +// are set to their defaults (if possible). type FSTabStageOptions struct { FileSystems []*FSTabEntry `json:"filesystems"` } func (FSTabStageOptions) isStageOptions() {} +// NewFSTabStage creates a now FSTabStage object func NewFSTabStage(options *FSTabStageOptions) *Stage { return &Stage{ Name: "org.osbuild.fstab", @@ -15,6 +21,8 @@ func NewFSTabStage(options *FSTabStageOptions) *Stage { } } +// An FSTabEntry represents one line in /etc/fstab. With the one exception +// that the the spec field must be represented as an UUID. type FSTabEntry struct { UUID uuid.UUID `json:"uuid"` VFSType string `json:"vfs_type"` @@ -24,6 +32,7 @@ type FSTabEntry struct { PassNo uint64 `json:"passno,omitempty"` } +// AddFilesystem adds one entry to and FSTabStageOptions object. func (options *FSTabStageOptions) AddFilesystem(id uuid.UUID, vfsType string, path string, opts string, freq uint64, passNo uint64) { options.FileSystems = append(options.FileSystems, &FSTabEntry{ UUID: id, diff --git a/internal/pipeline/grub2_stage.go b/internal/pipeline/grub2_stage.go index 4169bfb10..6c573b597 100644 --- a/internal/pipeline/grub2_stage.go +++ b/internal/pipeline/grub2_stage.go @@ -2,6 +2,14 @@ package pipeline import "github.com/google/uuid" +// The GRUB2StageOptions describes the bootloader configuration. +// +// The stage is responsible for installing all bootloader files in +// /boot as well as config files in /etc necessary for regenerating +// the configuration in /boot. +// +// Note that it is the role of an assembler to install any necessary +// bootloaders that are stored in the image outside of any filesystem. type GRUB2StageOptions struct { RootFilesystemUUID uuid.UUID `json:"root_fs_uuid"` BootFilesystemUUID uuid.UUID `json:"boot_fs_uuid,omitempty"` @@ -10,6 +18,8 @@ type GRUB2StageOptions struct { func (GRUB2StageOptions) isStageOptions() {} +// NewGRUB2StageOptions creates a new GRUB2StageOptions object. It sets the +// mandatory options. func NewGRUB2StageOptions(rootFilesystemUUID uuid.UUID) *GRUB2StageOptions { return &GRUB2StageOptions{ RootFilesystemUUID: rootFilesystemUUID, @@ -17,6 +27,7 @@ func NewGRUB2StageOptions(rootFilesystemUUID uuid.UUID) *GRUB2StageOptions { } +// NewGRUB2Stage creates a new GRUB2 stage object. func NewGRUB2Stage(options *GRUB2StageOptions) *Stage { return &Stage{ Name: "org.osbuild.grub2", @@ -24,10 +35,17 @@ func NewGRUB2Stage(options *GRUB2StageOptions) *Stage { } } +// SetRootFilesystemUUID sets the UUID of the filesystem containing /. +func (options *GRUB2StageOptions) SetRootFilesystemUUID(u uuid.UUID) { + options.RootFilesystemUUID = u +} + +// SetBootFilesystemUUID sets the UUID of the filesystem containing /boot. func (options *GRUB2StageOptions) SetBootFilesystemUUID(u uuid.UUID) { options.BootFilesystemUUID = u } +// SetKernelOptions sets the kernel options that should be passed at boot. func (options *GRUB2StageOptions) SetKernelOptions(kernelOptions string) { options.KernelOptions = kernelOptions } diff --git a/internal/pipeline/locale_stage.go b/internal/pipeline/locale_stage.go index adb245798..411431e20 100644 --- a/internal/pipeline/locale_stage.go +++ b/internal/pipeline/locale_stage.go @@ -1,17 +1,24 @@ package pipeline +// The LocaleStageOptions describes the image's locale. +// +// A locale is typically specified as language_[territory], where language +// is specified in ISO 639 and territory in ISO 3166. type LocaleStageOptions struct { Language string `json:"language"` } func (LocaleStageOptions) isStageOptions() {} +// NewLocaleStageOptions creates a new locale stage options object, with +// the mandatory fields set. func NewLocaleStageOptions(language string) *LocaleStageOptions { return &LocaleStageOptions{ Language: language, } } +// NewLocaleStage creates a new Locale Stage object. func NewLocaleStage(options *LocaleStageOptions) *Stage { return &Stage{ Name: "org.osbuild.locale", diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index d54da5fd8..f8e1804f8 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -1,111 +1,32 @@ +// Package pipeline provides primitives for representing and (un)marshalling +// OSBuild pipelines. package pipeline -import ( - "encoding/json" - "errors" -) - +// A Pipeline represents an OSBuild pipeline type Pipeline struct { - BuildPipeline *Pipeline `json:"build,omitempty"` - Stages []*Stage `json:"stages,omitempty"` - Assembler *Assembler `json:"assembler,omitempty"` -} - -type Stage struct { - Name string `json:"name"` - Options StageOptions `json:"options"` -} - -type StageOptions interface { - isStageOptions() -} - -type rawStage struct { - Name string `json:"name"` - Options json.RawMessage `json:"options"` -} - -func (stage *Stage) UnmarshalJSON(data []byte) error { - var rawStage rawStage - err := json.Unmarshal(data, &rawStage) - if err != nil { - return err - } - var options StageOptions - switch rawStage.Name { - case "org.osbuild.dnf": - options = new(DNFStageOptions) - case "org.osbuild.fix-bls": - options = new(FixBLSStageOptions) - case "org.osbuild.FSTab": - options = new(FSTabStageOptions) - case "org.osbuild.grub2": - options = new(GRUB2StageOptions) - case "org.osbuild.locale": - options = new(LocaleStageOptions) - case "org.osbuild.SELinux": - options = new(SELinuxStageOptions) - default: - return errors.New("unexpected stage name") - } - err = json.Unmarshal(rawStage.Options, options) - if err != nil { - return err - } - - stage.Name = rawStage.Name - stage.Options = options - - return nil -} - -type Assembler struct { - Name string `json:"name"` - Options AssemblerOptions `json:"options"` -} -type AssemblerOptions interface { - isAssemblerOptions() -} - -type rawAssembler struct { - Name string `json:"name"` - Options json.RawMessage `json:"options"` -} - -func (assembler *Assembler) UnmarshalJSON(data []byte) error { - var rawAssembler rawAssembler - err := json.Unmarshal(data, &rawAssembler) - if err != nil { - return err - } - var options AssemblerOptions - switch rawAssembler.Name { - case "org.osbuild.tar": - options = new(TarAssemblerOptions) - case "org.osbuild.qcow2": - options = new(QCOW2AssemblerOptions) - default: - return errors.New("unexpected assembler name") - } - err = json.Unmarshal(rawAssembler.Options, options) - if err != nil { - return err - } - - assembler.Name = rawAssembler.Name - assembler.Options = options - - return nil + // BuildPipeline describes how to create the build environment for the + // following stages and assembler. + BuildPipeline *Pipeline `json:"build,omitempty"` + // Sequence of stages that produce the filesystem tree, which is the + // payload of the produced image. + Stages []*Stage `json:"stages,omitempty"` + // Assembler that assembles the filesystem tree into the target image. + Assembler *Assembler `json:"assembler,omitempty"` } +// SetBuildPipeline sets the pipeline for generating the build environment for +// a pipeline. func (p *Pipeline) SetBuildPipeline(buildPipeline *Pipeline) { p.BuildPipeline = buildPipeline } +// AddStage appends a stage to the list of stages of a pipeline. The stages +// will be executed in the order they are appended. func (p *Pipeline) AddStage(stage *Stage) { p.Stages = append(p.Stages, stage) } +// SetAssembler sets the assembler for a pipeline. func (p *Pipeline) SetAssembler(assembler *Assembler) { p.Assembler = assembler } diff --git a/internal/pipeline/qcow2_assembler.go b/internal/pipeline/qcow2_assembler.go index a047b535d..73a2aed9a 100644 --- a/internal/pipeline/qcow2_assembler.go +++ b/internal/pipeline/qcow2_assembler.go @@ -2,23 +2,37 @@ package pipeline import "github.com/google/uuid" -type QCOW2AssemblerOptions struct { +// QEMUAssemblerOptions desrcibe how to assemble a tree into an image using qemu. +// +// The assembler creates an image of a the given size, adds a GRUB2 bootloader +// and a DOS partition table to it with the given PTUUID containing one ext4 +// root partition with the given filesystem UUID and installs the filesystem +// tree into it. Finally, the image is converted into the target format and +// stored with the given filename. +type QEMUAssemblerOptions struct { + Format string `json:"format"` Filename string `json:"filename"` + PTUUID string `json:"ptuuid"` RootFilesystemUUDI uuid.UUID `json:"root_fs_uuid"` Size uint64 `json:"size"` } -func (QCOW2AssemblerOptions) isAssemblerOptions() {} +func (QEMUAssemblerOptions) isAssemblerOptions() {} -func NewQCOW2AssemblerOptions(filename string, rootFilesystemUUID uuid.UUID, size uint64) *QCOW2AssemblerOptions { - return &QCOW2AssemblerOptions{ +// NewQEMUAssemblerOptions creates a now QEMUAssemblerOptions object, with all the mandatory +// fields set. +func NewQEMUAssemblerOptions(format string, ptUUID string, filename string, rootFilesystemUUID uuid.UUID, size uint64) *QEMUAssemblerOptions { + return &QEMUAssemblerOptions{ + Format: format, + PTUUID: ptUUID, Filename: filename, RootFilesystemUUDI: rootFilesystemUUID, Size: size, } } -func NewQCOW2Assembler(options *QCOW2AssemblerOptions) *Assembler { +// NewQEMUAssembler creates a new QEMU Assembler object. +func NewQEMUAssembler(options *QEMUAssemblerOptions) *Assembler { return &Assembler{ Name: "org.osbuild.qcow2", Options: options, diff --git a/internal/pipeline/selinux_stage.go b/internal/pipeline/selinux_stage.go index 5f51da577..747169e30 100644 --- a/internal/pipeline/selinux_stage.go +++ b/internal/pipeline/selinux_stage.go @@ -1,17 +1,24 @@ package pipeline +// The SELinuxStageOptions describe how to apply selinux labels. +// +// A file contexts configuration file is sepcified that describes +// the filesystem labels to apply to the image. type SELinuxStageOptions struct { FileContexts string `json:"file_contexts"` } func (SELinuxStageOptions) isStageOptions() {} +// NewSELinuxStageOptions creates a new SELinuxStaeOptions object, with +// the mandatory fields set. func NewSELinuxStageOptions(fileContexts string) *SELinuxStageOptions { return &SELinuxStageOptions{ FileContexts: fileContexts, } } +// NewSELinuxStage creates a new SELinux Stage object. func NewSELinuxStage(options *SELinuxStageOptions) *Stage { return &Stage{ Name: "org.osbuild.selinux", diff --git a/internal/pipeline/stage.go b/internal/pipeline/stage.go new file mode 100644 index 000000000..b7be4ad7a --- /dev/null +++ b/internal/pipeline/stage.go @@ -0,0 +1,63 @@ +package pipeline + +import ( + "encoding/json" + "errors" +) + +// A Stage transforms a filesystem tree. +type Stage struct { + // Well-known name in reverse domain-name notation, uniquely identifying + // the stage type. + Name string `json:"name"` + // Stage-type specific options fully determining the operations of the + // stage. + Options StageOptions `json:"options"` +} + +// StageOptions specify the operations of a given stage-type. +type StageOptions interface { + isStageOptions() +} + +type rawStage struct { + Name string `json:"name"` + Options json.RawMessage `json:"options"` +} + +// UnmarshalJSON unmarshals JSON into a Stage object. Each type of stage has +// a custom unmarshaller for its options, selected based on the stage name. +func (stage *Stage) UnmarshalJSON(data []byte) error { + var rawStage rawStage + err := json.Unmarshal(data, &rawStage) + if err != nil { + return err + } + var options StageOptions + switch rawStage.Name { + case "org.osbuild.dnf": + options = new(DNFStageOptions) + case "org.osbuild.fix-bls": + // TODO: verify that we can unmarshall this also if "options" is omitted + options = new(FixBLSStageOptions) + case "org.osbuild.fstab": + options = new(FSTabStageOptions) + case "org.osbuild.grub2": + options = new(GRUB2StageOptions) + case "org.osbuild.locale": + options = new(LocaleStageOptions) + case "org.osbuild.selinux": + options = new(SELinuxStageOptions) + default: + return errors.New("unexpected stage name") + } + err = json.Unmarshal(rawStage.Options, options) + if err != nil { + return err + } + + stage.Name = rawStage.Name + stage.Options = options + + return nil +} diff --git a/internal/pipeline/tar_assembler.go b/internal/pipeline/tar_assembler.go index 2aaaba556..c0e79349a 100644 --- a/internal/pipeline/tar_assembler.go +++ b/internal/pipeline/tar_assembler.go @@ -1,5 +1,9 @@ package pipeline +// TarAssemblerOptions desrcibe how to assemble a tree into a tar ball. +// +// The assembler tars and optionally compresses the tree using the provided +// compression type, and stores the output with the given filename. type TarAssemblerOptions struct { Filename string `json:"filename"` Compression string `json:"compression,omitempty"` @@ -7,12 +11,15 @@ type TarAssemblerOptions struct { func (TarAssemblerOptions) isAssemblerOptions() {} +// NewTarAssemblerOptions creates a new TarAssemblerOptions object, with the +// mandatory options set. func NewTarAssemblerOptions(filename string) *TarAssemblerOptions { return &TarAssemblerOptions{ Filename: filename, } } +// NewTarAssembler creates a new Tar Assembler object. func NewTarAssembler(options *TarAssemblerOptions) *Assembler { return &Assembler{ Name: "org.osbuild.tar", @@ -20,6 +27,8 @@ func NewTarAssembler(options *TarAssemblerOptions) *Assembler { } } +// SetCompression sets the compression type for a given TarAssemblerOptions +// object. func (options *TarAssemblerOptions) SetCompression(compression string) { options.Compression = compression }