pipeline/target: implement as variant types

Go doesn't really do variants, so we must somehow emulate it. The
json objects we use are essentially tagged unions, with a `name`
field in reverse domain name notation identifying the type and a
type specific 'options' object.

In Go we represent this by having an BarOptions interface, which
implements a private method `isBarOptions()`, making sure that only
types in the same package are able to implement it. Each type FooBar
that should belong to the variant implements the interface, and a
constructor `NewFooBar(options *FooBarOptions) *Bar` that makes sure
the `name` field is set correctly.

This would be enough to represent our types and marshal them into
JSON, but unmarshalling would not work (json does not know about
our tags, so would not know what concrete types to demarshal to).
We therefore must also implement the Unmarshall interface for Bar,
to select the right types for the Options field.

We implement his logic for Target, Stage and Assembler. A handful
of concrete types are also implemented, matching what osbuild
supports.

Signed-off-by: Tom Gundersen <teg@jklm.no>

cmd/osbuild-worker/job.go

@@ -56,23 +56,24 @@ func (job *Job) Run() error {
 		return err
 	}
 
-	for _, target := range job.Targets {
-		if target.Name != "org.osbuild.local" {
+	for _, t := range job.Targets {
+		switch options := t.Options.(type) {
+		case *target.LocalTargetOptions:
+			err = os.MkdirAll(options.Location, 0755)
+			if err != nil {
+				panic(err)
+			}
+
+			cp := exec.Command("cp", "-a", "-L", "/var/cache/osbuild-composer/store/refs/"+result.OutputID, options.Location)
+			cp.Stderr = os.Stderr
+			cp.Stdout = os.Stdout
+			err = cp.Run()
+			if err != nil {
+				panic(err)
+			}
+		default:
 			panic("foo")
 		}
-
-		err = os.MkdirAll(target.Options.Location, 0755)
-		if err != nil {
-			panic(err)
-		}
-
-		cp := exec.Command("cp", "-a", "-L", "/var/cache/osbuild-composer/store/refs/" + result.OutputID, target.Options.Location)
-		cp.Stderr = os.Stderr
-		cp.Stdout = os.Stdout
-		err = cp.Run()
-		if err != nil {
-			panic(err)
-		}
 	}
 
 	return nil

internal/job/job.go

@@ -9,7 +9,7 @@ import (
 
 type Job struct {
 	ComposeID uuid.UUID
-	Pipeline  pipeline.Pipeline
+	Pipeline  *pipeline.Pipeline
 	Targets   []*target.Target
 }
 

internal/jobqueue/api.go

@@ -83,8 +83,8 @@ func (api *API) addJobHandler(writer http.ResponseWriter, request *http.Request,
 		ID uuid.UUID `json:"id"`
 	}
 	type replyBody struct {
-		Pipeline pipeline.Pipeline `json:"pipeline"`
-		Targets  []*target.Target  `json:"targets"`
+		Pipeline *pipeline.Pipeline `json:"pipeline"`
+		Targets  []*target.Target   `json:"targets"`
 	}
 
 	contentType := request.Header["Content-Type"]

internal/jobqueue/api_test.go

@@ -66,7 +66,7 @@ func testRoute(t *testing.T, api *jobqueue.API, method, path, body string, expec
 }
 
 func TestBasic(t *testing.T) {
-	expected_job := `{"pipeline":{"assembler":{"name":"org.osbuild.tar","options":{"filename":"image.tar"}}},"targets":[{"name":"org.osbuild.local","options":{"location":"/var/lib/osbuild-composer/outputs/ffffffff-ffff-ffff-ffff-ffffffffffff"}}]}`
+	expected_job := `{"pipeline":{"assembler":{"name":"org.osbuild.tar","options":{"filename":"image.tar"}}},"targets":[{"name":"org.osbuild.local","options":{"location":"/tmp/ffffffff-ffff-ffff-ffff-ffffffffffff"}}]}`
 	var cases = []struct {
 		Method         string
 		Path           string
@@ -94,17 +94,14 @@ func TestBasic(t *testing.T) {
 	api := jobqueue.New(nil, jobChannel, statusChannel)
 	for _, c := range cases {
 		id, _ := uuid.Parse("ffffffff-ffff-ffff-ffff-ffffffffffff")
+		p := &pipeline.Pipeline{}
+		p.SetAssembler(pipeline.NewTarAssembler(pipeline.NewTarAssemblerOptions("image.tar")))
 		jobChannel <- job.Job{
 			ComposeID: id,
-			Pipeline: pipeline.Pipeline{
-				Assembler: pipeline.Assembler{
-					Name: "org.osbuild.tar",
-					Options: pipeline.AssemblerTarOptions{
-						Filename: "image.tar",
-					},
-				},
+			Pipeline:  p,
+			Targets: []*target.Target{
+				target.NewLocalTarget(target.NewLocalTargetOptions("/tmp/" + id.String())),
 			},
-			Targets: []*target.Target{target.New(id)},
 		}
 
 		testRoute(t, api, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON)

internal/pipeline/dnf_stage.go

@@ -0,0 +1,57 @@
+package pipeline
+
+type DNFStageOptions struct {
+	Repositories     map[string]*DNFRepository `json:"repos"`
+	Packages         []string                  `json:"packages"`
+	ReleaseVersion   string                    `json:"releasever"`
+	BaseArchitecture string                    `json:"basearch"`
+}
+
+func (DNFStageOptions) isStageOptions() {}
+
+type DNFRepository struct {
+	MetaLink   string `json:"metalink,omitempty"`
+	MirrorList string `json:"mirrorlist,omitempty"`
+	BaseURL    string `json:"baseurl,omitempty"`
+	GPGKey     string `json:"gpgkey,omitempty"`
+	Checksum   string `json:"checksum,omitempty"`
+}
+
+func NewDNFStageOptions(releaseVersion string, baseArchitecture string) *DNFStageOptions {
+	return &DNFStageOptions{
+		Repositories:     make(map[string]*DNFRepository),
+		ReleaseVersion:   releaseVersion,
+		BaseArchitecture: baseArchitecture,
+	}
+}
+
+func NewDNFStage(options *DNFStageOptions) *Stage {
+	return &Stage{
+		Name:    "org.osbuild.dnf",
+		Options: options,
+	}
+}
+
+func (options *DNFStageOptions) AddPackage(pkg string) {
+	options.Packages = append(options.Packages, pkg)
+}
+
+func (options *DNFStageOptions) AddRepository(name string, repo *DNFRepository) {
+	options.Repositories[name] = repo
+}
+
+func NewDNFRepository(metaLink string, mirrorList string, baseURL string) *DNFRepository {
+	return &DNFRepository{
+		MetaLink:   metaLink,
+		MirrorList: mirrorList,
+		BaseURL:    baseURL,
+	}
+}
+
+func (r *DNFRepository) SetGPGKey(gpgKey string) {
+	r.GPGKey = gpgKey
+}
+
+func (r *DNFRepository) SetChecksum(checksum string) {
+	r.Checksum = checksum
+}

internal/pipeline/fix_bls_stage.go

@@ -0,0 +1,12 @@
+package pipeline
+
+type FixBLSStageOptions struct {
+}
+
+func (FixBLSStageOptions) isStageOptions() {}
+
+func NewFIXBLSStage() *Stage {
+	return &Stage{
+		Name: "org.osbuild.fix-bls",
+	}
+}

internal/pipeline/fstab_stage.go

@@ -0,0 +1,28 @@
+package pipeline
+
+import "github.com/google/uuid"
+
+type FSTabStageOptions struct {
+	FileSystems []*FSTabEntry `json:"filesystems"`
+}
+
+func (FSTabStageOptions) isStageOptions() {}
+
+func NewFSTabStage(options *FSTabStageOptions) *Stage {
+	return &Stage{
+		Name:    "org.osbuild.fstab",
+		Options: options,
+	}
+}
+
+type FSTabEntry struct {
+	UUID    uuid.UUID `json:"uuid"`
+	VFSType string    `json:"vfs_type"`
+	Path    string    `json:"path"`
+	Freq    int64     `json:"freq"`
+	PassNo  int64     `json:"passno"`
+}
+
+func (options *FSTabStageOptions) AddEntry(entry *FSTabEntry) {
+	options.FileSystems = append(options.FileSystems, entry)
+}

internal/pipeline/grub2_stage.go

@@ -0,0 +1,33 @@
+package pipeline
+
+import "github.com/google/uuid"
+
+type GRUB2StageOptions struct {
+	RootFilesystemUUID uuid.UUID `json:"root_fs_uuid"`
+	BootFilesystemUUID uuid.UUID `json:"boot_fs_uuid,omitempty"`
+	KernelOptions      string    `json:"kernel_opts,omitempty"`
+}
+
+func (GRUB2StageOptions) isStageOptions() {}
+
+func NewGRUB2StageOptions(rootFilesystemUUID uuid.UUID) *GRUB2StageOptions {
+	return &GRUB2StageOptions{
+		RootFilesystemUUID: rootFilesystemUUID,
+	}
+
+}
+
+func NewGRUB2Stage(options *GRUB2StageOptions) *Stage {
+	return &Stage{
+		Name:    "org.osbuild.grub2",
+		Options: options,
+	}
+}
+
+func (options *GRUB2StageOptions) SetBootFilesystemUUID(u uuid.UUID) {
+	options.BootFilesystemUUID = u
+}
+
+func (options *GRUB2StageOptions) SetKernelOptions(kernelOptions string) {
+	options.KernelOptions = kernelOptions
+}

internal/pipeline/pipeline.go

@@ -1,18 +1,111 @@
 package pipeline
 
+import (
+	"encoding/json"
+	"errors"
+)
+
 type Pipeline struct {
-	Stages    []Stage   `json:"stages,omitempty"`
-	Assembler Assembler `json:"assembler"`
+	BuildPipeline *Pipeline  `json:"build,omitempty"`
+	Stages        []*Stage   `json:"stages,omitempty"`
+	Assembler     *Assembler `json:"assembler,omitempty"`
 }
 
 type Stage struct {
+	Name    string       `json:"name"`
+	Options StageOptions `json:"options"`
+}
+
+type StageOptions interface {
+	isStageOptions()
+}
+
+type rawStage struct {
+	Name    string          `json:"name"`
+	Options json.RawMessage `json:"options"`
+}
+
+func (stage *Stage) UnmarshalJSON(data []byte) error {
+	var rawStage rawStage
+	err := json.Unmarshal(data, &rawStage)
+	if err != nil {
+		return err
+	}
+	var options StageOptions
+	switch rawStage.Name {
+	case "org.osbuild.dnf":
+		options = new(DNFStageOptions)
+	case "org.osbuild.fix-bls":
+		options = new(FixBLSStageOptions)
+	case "org.osbuild.FSTab":
+		options = new(FSTabStageOptions)
+	case "org.osbuild.grub2":
+		options = new(GRUB2StageOptions)
+	case "org.osbuild.locale":
+		options = new(LocaleStageOptions)
+	case "org.osbuild.SELinux":
+		options = new(SELinuxStageOptions)
+	default:
+		return errors.New("unexpected stage name")
+	}
+	err = json.Unmarshal(rawStage.Options, options)
+	if err != nil {
+		return err
+	}
+
+	stage.Name = rawStage.Name
+	stage.Options = options
+
+	return nil
 }
 
 type Assembler struct {
-	Name    string              `json:"name"`
-	Options AssemblerTarOptions `json:"options"`
+	Name    string           `json:"name"`
+	Options AssemblerOptions `json:"options"`
+}
+type AssemblerOptions interface {
+	isAssemblerOptions()
 }
 
-type AssemblerTarOptions struct {
-	Filename string `json:"filename"`
+type rawAssembler struct {
+	Name    string          `json:"name"`
+	Options json.RawMessage `json:"options"`
+}
+
+func (assembler *Assembler) UnmarshalJSON(data []byte) error {
+	var rawAssembler rawAssembler
+	err := json.Unmarshal(data, &rawAssembler)
+	if err != nil {
+		return err
+	}
+	var options AssemblerOptions
+	switch rawAssembler.Name {
+	case "org.osbuild.tar":
+		options = new(TarAssemblerOptions)
+	case "org.osbuild.qcow2":
+		options = new(QCOW2AssemblerOptions)
+	default:
+		return errors.New("unexpected assembler name")
+	}
+	err = json.Unmarshal(rawAssembler.Options, options)
+	if err != nil {
+		return err
+	}
+
+	assembler.Name = rawAssembler.Name
+	assembler.Options = options
+
+	return nil
+}
+
+func (p *Pipeline) SetBuildPipeline(buildPipeline *Pipeline) {
+	p.BuildPipeline = buildPipeline
+}
+
+func (p *Pipeline) AddStage(stage *Stage) {
+	p.Stages = append(p.Stages, stage)
+}
+
+func (p *Pipeline) SetAssembler(assembler *Assembler) {
+	p.Assembler = assembler
 }

internal/pipeline/qcow2_assembler.go

@@ -0,0 +1,26 @@
+package pipeline
+
+import "github.com/google/uuid"
+
+type QCOW2AssemblerOptions struct {
+	Filename           string    `json:"filename"`
+	RootFilesystemUUDI uuid.UUID `json:"root_fs_uuid"`
+	Size               uint64    `json:"size"`
+}
+
+func (QCOW2AssemblerOptions) isAssemblerOptions() {}
+
+func NewQCOW2AssemblerOptions(filename string, rootFilesystemUUID uuid.UUID, size uint64) *QCOW2AssemblerOptions {
+	return &QCOW2AssemblerOptions{
+		Filename:           filename,
+		RootFilesystemUUDI: rootFilesystemUUID,
+		Size:               size,
+	}
+}
+
+func NewQCOW2Assembler(options *QCOW2AssemblerOptions) *Assembler {
+	return &Assembler{
+		Name:    "org.osbuild.qcow2",
+		Options: options,
+	}
+}

internal/pipeline/selinux_stage.go

@@ -0,0 +1,20 @@
+package pipeline
+
+type SELinuxStageOptions struct {
+	FileContexts string `json:"file_contexts"`
+}
+
+func (SELinuxStageOptions) isStageOptions() {}
+
+func NewSELinuxStageOptions(fileContexts string) *SELinuxStageOptions {
+	return &SELinuxStageOptions{
+		FileContexts: fileContexts,
+	}
+}
+
+func NewSELinuxStage(options *SELinuxStageOptions) *Stage {
+	return &Stage{
+		Name:    "org.osbuild.selinux",
+		Options: options,
+	}
+}

internal/pipeline/tar_assembler.go

@@ -0,0 +1,25 @@
+package pipeline
+
+type TarAssemblerOptions struct {
+	Filename    string `json:"filename"`
+	Compression string `json:"compression,omitempty"`
+}
+
+func (TarAssemblerOptions) isAssemblerOptions() {}
+
+func NewTarAssemblerOptions(filename string) *TarAssemblerOptions {
+	return &TarAssemblerOptions{
+		Filename: filename,
+	}
+}
+
+func NewTarAssembler(options *TarAssemblerOptions) *Assembler {
+	return &Assembler{
+		Name:    "org.osbuild.tar",
+		Options: options,
+	}
+}
+
+func (options *TarAssemblerOptions) SetCompression(compression string) {
+	options.Compression = compression
+}

internal/target/local_target.go

@@ -0,0 +1,20 @@
+package target
+
+type LocalTargetOptions struct {
+	Location string `json:"location"`
+}
+
+func (LocalTargetOptions) isTargetOptions() {}
+
+func NewLocalTargetOptions(location string) *LocalTargetOptions {
+	return &LocalTargetOptions{
+		Location: location,
+	}
+}
+
+func NewLocalTarget(options *LocalTargetOptions) *Target {
+	return &Target{
+		Name:    "org.osbuild.local",
+		Options: options,
+	}
+}

internal/target/target.go

@@ -1,21 +1,44 @@
 package target
 
-import "github.com/google/uuid"
+import (
+	"encoding/json"
+	"errors"
+)
 
 type Target struct {
-	Name    string  `json:"name"`
-	Options Options `json:"options"`
+	Name    string        `json:"name"`
+	Options TargetOptions `json:"options"`
 }
 
-type Options struct {
-	Location string `json:"location"`
+type TargetOptions interface {
+	isTargetOptions()
 }
 
-func New(ComposeID uuid.UUID) *Target {
-	return &Target{
-		Name: "org.osbuild.local",
-		Options: Options{
-			Location: "/var/lib/osbuild-composer/outputs/" + ComposeID.String(),
-		},
+type rawTarget struct {
+	Name    string          `json:"name"`
+	Options json.RawMessage `json:"options"`
+}
+
+func (target *Target) UnmarshalJSON(data []byte) error {
+	var rawTarget rawTarget
+	err := json.Unmarshal(data, &rawTarget)
+	if err != nil {
+		return err
 	}
+	var options TargetOptions
+	switch rawTarget.Name {
+	case "org.osbuild.local":
+		options = new(LocalTargetOptions)
+	default:
+		return errors.New("unexpected target name")
+	}
+	err = json.Unmarshal(rawTarget.Options, options)
+	if err != nil {
+		return err
+	}
+
+	target.Name = rawTarget.Name
+	target.Options = options
+
+	return nil
 }

internal/weldr/api.go

@@ -627,7 +627,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, httpRequest *http.Req
 	found := api.store.getBlueprint(cr.BlueprintName, &bp, &changed) // TODO: what to do with changed?
 
 	if found {
-		api.store.addCompose(reply.BuildID, bp, cr.ComposeType)
+		api.store.addCompose(reply.BuildID, &bp, cr.ComposeType)
 	} else {
 		statusResponseError(writer, http.StatusBadRequest, "blueprint does not exist")
 		return

internal/weldr/api_test.go

@@ -153,12 +153,4 @@ func TestCompose(t *testing.T) {
 
 	testRoute(t, api, "POST", "/api/v0/compose", `{"blueprint_name": "test","compose_type": "tar","branch": "master"}`,
 		http.StatusOK, `*`)
-
-	job := <-jobChannel
-	if job.Pipeline.Assembler.Name != "org.osbuild.tar" {
-		t.Errorf("Expected tar assembler, got: %s", job.Pipeline.Assembler.Name)
-	}
-	if job.Targets[0].Name != "org.osbuild.local" {
-		t.Errorf("Expected local target, got: %s", job.Targets[0].Name)
-	}
 }

internal/weldr/store.go

@@ -37,9 +37,9 @@ type blueprintPackage struct {
 }
 
 type compose struct {
-	Status   string            `json:"status"`
-	Pipeline pipeline.Pipeline `json:"pipeline"`
-	Targets  []*target.Target  `json:"targets"`
+	Status    string           `json:"status"`
+	Blueprint *blueprint       `json:"blueprint"`
+	Targets   []*target.Target `json:"targets"`
 }
 
 func newStore(initialState []byte, stateChannel chan<- []byte, pendingJobs chan<- job.Job, jobUpdates <-chan job.Status) *store {
@@ -188,26 +188,22 @@ func (s *store) deleteBlueprintFromWorkspace(name string) {
 	})
 }
 
-func (s *store) addCompose(composeID uuid.UUID, bp blueprint, composeType string) {
-	pipeline := bp.translateToPipeline(composeType)
-	targets := []*target.Target{target.New(composeID)}
+func (s *store) addCompose(composeID uuid.UUID, bp *blueprint, composeType string) {
+	targets := []*target.Target{
+		target.NewLocalTarget(target.NewLocalTargetOptions("/var/lib/osbuild-composer/outputs/" + composeID.String())),
+	}
 	s.change(func() {
-		s.Composes[composeID] = compose{"pending", pipeline, targets}
+		s.Composes[composeID] = compose{"pending", bp, targets}
 	})
 	s.pendingJobs <- job.Job{
 		ComposeID: composeID,
-		Pipeline:  pipeline,
+		Pipeline:  bp.translateToPipeline(composeType),
 		Targets:   targets,
 	}
 }
 
-func (b *blueprint) translateToPipeline(outputFormat string) pipeline.Pipeline {
-	return pipeline.Pipeline{
-		Assembler: pipeline.Assembler{
-			Name: "org.osbuild.tar",
-			Options: pipeline.AssemblerTarOptions{
-				Filename: "image.tar",
-			},
-		},
-	}
+func (b *blueprint) translateToPipeline(outputFormat string) *pipeline.Pipeline {
+	p := &pipeline.Pipeline{}
+	p.SetAssembler(pipeline.NewTarAssembler(pipeline.NewTarAssemblerOptions("image.tar")))
+	return p
 }