From 7625d26ff547b40e7efc1f407c1c38aeb1f8bc3e Mon Sep 17 00:00:00 2001 From: Tom Gundersen Date: Fri, 27 Sep 2019 23:47:11 +0200 Subject: [PATCH] pipeline/target: implement as variant types Go doesn't really do variants, so we must somehow emulate it. The json objects we use are essentially tagged unions, with a `name` field in reverse domain name notation identifying the type and a type specific 'options' object. In Go we represent this by having an BarOptions interface, which implements a private method `isBarOptions()`, making sure that only types in the same package are able to implement it. Each type FooBar that should belong to the variant implements the interface, and a constructor `NewFooBar(options *FooBarOptions) *Bar` that makes sure the `name` field is set correctly. This would be enough to represent our types and marshal them into JSON, but unmarshalling would not work (json does not know about our tags, so would not know what concrete types to demarshal to). We therefore must also implement the Unmarshall interface for Bar, to select the right types for the Options field. We implement his logic for Target, Stage and Assembler. A handful of concrete types are also implemented, matching what osbuild supports. Signed-off-by: Tom Gundersen --- cmd/osbuild-worker/job.go | 31 ++++---- internal/job/job.go | 2 +- internal/jobqueue/api.go | 4 +- internal/jobqueue/api_test.go | 15 ++-- internal/pipeline/dnf_stage.go | 57 +++++++++++++++ internal/pipeline/fix_bls_stage.go | 12 +++ internal/pipeline/fstab_stage.go | 28 +++++++ internal/pipeline/grub2_stage.go | 33 +++++++++ internal/pipeline/pipeline.go | 105 +++++++++++++++++++++++++-- internal/pipeline/qcow2_assembler.go | 26 +++++++ internal/pipeline/selinux_stage.go | 20 +++++ internal/pipeline/tar_assembler.go | 25 +++++++ internal/target/local_target.go | 20 +++++ internal/target/target.go | 45 +++++++++--- internal/weldr/api.go | 2 +- internal/weldr/api_test.go | 8 -- internal/weldr/store.go | 30 ++++---- 17 files changed, 393 insertions(+), 70 deletions(-) create mode 100644 internal/pipeline/dnf_stage.go create mode 100644 internal/pipeline/fix_bls_stage.go create mode 100644 internal/pipeline/fstab_stage.go create mode 100644 internal/pipeline/grub2_stage.go create mode 100644 internal/pipeline/qcow2_assembler.go create mode 100644 internal/pipeline/selinux_stage.go create mode 100644 internal/pipeline/tar_assembler.go create mode 100644 internal/target/local_target.go diff --git a/cmd/osbuild-worker/job.go b/cmd/osbuild-worker/job.go index 1a64c41a7..53d7c8c93 100644 --- a/cmd/osbuild-worker/job.go +++ b/cmd/osbuild-worker/job.go @@ -56,23 +56,24 @@ func (job *Job) Run() error { return err } - for _, target := range job.Targets { - if target.Name != "org.osbuild.local" { + for _, t := range job.Targets { + switch options := t.Options.(type) { + case *target.LocalTargetOptions: + err = os.MkdirAll(options.Location, 0755) + if err != nil { + panic(err) + } + + cp := exec.Command("cp", "-a", "-L", "/var/cache/osbuild-composer/store/refs/"+result.OutputID, options.Location) + cp.Stderr = os.Stderr + cp.Stdout = os.Stdout + err = cp.Run() + if err != nil { + panic(err) + } + default: panic("foo") } - - err = os.MkdirAll(target.Options.Location, 0755) - if err != nil { - panic(err) - } - - cp := exec.Command("cp", "-a", "-L", "/var/cache/osbuild-composer/store/refs/" + result.OutputID, target.Options.Location) - cp.Stderr = os.Stderr - cp.Stdout = os.Stdout - err = cp.Run() - if err != nil { - panic(err) - } } return nil diff --git a/internal/job/job.go b/internal/job/job.go index 21b485e08..15248ba82 100644 --- a/internal/job/job.go +++ b/internal/job/job.go @@ -9,7 +9,7 @@ import ( type Job struct { ComposeID uuid.UUID - Pipeline pipeline.Pipeline + Pipeline *pipeline.Pipeline Targets []*target.Target } diff --git a/internal/jobqueue/api.go b/internal/jobqueue/api.go index 24a0a7ff1..1e0f97551 100644 --- a/internal/jobqueue/api.go +++ b/internal/jobqueue/api.go @@ -83,8 +83,8 @@ func (api *API) addJobHandler(writer http.ResponseWriter, request *http.Request, ID uuid.UUID `json:"id"` } type replyBody struct { - Pipeline pipeline.Pipeline `json:"pipeline"` - Targets []*target.Target `json:"targets"` + Pipeline *pipeline.Pipeline `json:"pipeline"` + Targets []*target.Target `json:"targets"` } contentType := request.Header["Content-Type"] diff --git a/internal/jobqueue/api_test.go b/internal/jobqueue/api_test.go index 4a33da279..76fafa2d2 100644 --- a/internal/jobqueue/api_test.go +++ b/internal/jobqueue/api_test.go @@ -66,7 +66,7 @@ func testRoute(t *testing.T, api *jobqueue.API, method, path, body string, expec } func TestBasic(t *testing.T) { - expected_job := `{"pipeline":{"assembler":{"name":"org.osbuild.tar","options":{"filename":"image.tar"}}},"targets":[{"name":"org.osbuild.local","options":{"location":"/var/lib/osbuild-composer/outputs/ffffffff-ffff-ffff-ffff-ffffffffffff"}}]}` + expected_job := `{"pipeline":{"assembler":{"name":"org.osbuild.tar","options":{"filename":"image.tar"}}},"targets":[{"name":"org.osbuild.local","options":{"location":"/tmp/ffffffff-ffff-ffff-ffff-ffffffffffff"}}]}` var cases = []struct { Method string Path string @@ -94,17 +94,14 @@ func TestBasic(t *testing.T) { api := jobqueue.New(nil, jobChannel, statusChannel) for _, c := range cases { id, _ := uuid.Parse("ffffffff-ffff-ffff-ffff-ffffffffffff") + p := &pipeline.Pipeline{} + p.SetAssembler(pipeline.NewTarAssembler(pipeline.NewTarAssemblerOptions("image.tar"))) jobChannel <- job.Job{ ComposeID: id, - Pipeline: pipeline.Pipeline{ - Assembler: pipeline.Assembler{ - Name: "org.osbuild.tar", - Options: pipeline.AssemblerTarOptions{ - Filename: "image.tar", - }, - }, + Pipeline: p, + Targets: []*target.Target{ + target.NewLocalTarget(target.NewLocalTargetOptions("/tmp/" + id.String())), }, - Targets: []*target.Target{target.New(id)}, } testRoute(t, api, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON) diff --git a/internal/pipeline/dnf_stage.go b/internal/pipeline/dnf_stage.go new file mode 100644 index 000000000..eb9743959 --- /dev/null +++ b/internal/pipeline/dnf_stage.go @@ -0,0 +1,57 @@ +package pipeline + +type DNFStageOptions struct { + Repositories map[string]*DNFRepository `json:"repos"` + Packages []string `json:"packages"` + ReleaseVersion string `json:"releasever"` + BaseArchitecture string `json:"basearch"` +} + +func (DNFStageOptions) isStageOptions() {} + +type DNFRepository struct { + MetaLink string `json:"metalink,omitempty"` + MirrorList string `json:"mirrorlist,omitempty"` + BaseURL string `json:"baseurl,omitempty"` + GPGKey string `json:"gpgkey,omitempty"` + Checksum string `json:"checksum,omitempty"` +} + +func NewDNFStageOptions(releaseVersion string, baseArchitecture string) *DNFStageOptions { + return &DNFStageOptions{ + Repositories: make(map[string]*DNFRepository), + ReleaseVersion: releaseVersion, + BaseArchitecture: baseArchitecture, + } +} + +func NewDNFStage(options *DNFStageOptions) *Stage { + return &Stage{ + Name: "org.osbuild.dnf", + Options: options, + } +} + +func (options *DNFStageOptions) AddPackage(pkg string) { + options.Packages = append(options.Packages, pkg) +} + +func (options *DNFStageOptions) AddRepository(name string, repo *DNFRepository) { + options.Repositories[name] = repo +} + +func NewDNFRepository(metaLink string, mirrorList string, baseURL string) *DNFRepository { + return &DNFRepository{ + MetaLink: metaLink, + MirrorList: mirrorList, + BaseURL: baseURL, + } +} + +func (r *DNFRepository) SetGPGKey(gpgKey string) { + r.GPGKey = gpgKey +} + +func (r *DNFRepository) SetChecksum(checksum string) { + r.Checksum = checksum +} diff --git a/internal/pipeline/fix_bls_stage.go b/internal/pipeline/fix_bls_stage.go new file mode 100644 index 000000000..7ac3c3b86 --- /dev/null +++ b/internal/pipeline/fix_bls_stage.go @@ -0,0 +1,12 @@ +package pipeline + +type FixBLSStageOptions struct { +} + +func (FixBLSStageOptions) isStageOptions() {} + +func NewFIXBLSStage() *Stage { + return &Stage{ + Name: "org.osbuild.fix-bls", + } +} diff --git a/internal/pipeline/fstab_stage.go b/internal/pipeline/fstab_stage.go new file mode 100644 index 000000000..600599bc5 --- /dev/null +++ b/internal/pipeline/fstab_stage.go @@ -0,0 +1,28 @@ +package pipeline + +import "github.com/google/uuid" + +type FSTabStageOptions struct { + FileSystems []*FSTabEntry `json:"filesystems"` +} + +func (FSTabStageOptions) isStageOptions() {} + +func NewFSTabStage(options *FSTabStageOptions) *Stage { + return &Stage{ + Name: "org.osbuild.fstab", + Options: options, + } +} + +type FSTabEntry struct { + UUID uuid.UUID `json:"uuid"` + VFSType string `json:"vfs_type"` + Path string `json:"path"` + Freq int64 `json:"freq"` + PassNo int64 `json:"passno"` +} + +func (options *FSTabStageOptions) AddEntry(entry *FSTabEntry) { + options.FileSystems = append(options.FileSystems, entry) +} diff --git a/internal/pipeline/grub2_stage.go b/internal/pipeline/grub2_stage.go new file mode 100644 index 000000000..4169bfb10 --- /dev/null +++ b/internal/pipeline/grub2_stage.go @@ -0,0 +1,33 @@ +package pipeline + +import "github.com/google/uuid" + +type GRUB2StageOptions struct { + RootFilesystemUUID uuid.UUID `json:"root_fs_uuid"` + BootFilesystemUUID uuid.UUID `json:"boot_fs_uuid,omitempty"` + KernelOptions string `json:"kernel_opts,omitempty"` +} + +func (GRUB2StageOptions) isStageOptions() {} + +func NewGRUB2StageOptions(rootFilesystemUUID uuid.UUID) *GRUB2StageOptions { + return &GRUB2StageOptions{ + RootFilesystemUUID: rootFilesystemUUID, + } + +} + +func NewGRUB2Stage(options *GRUB2StageOptions) *Stage { + return &Stage{ + Name: "org.osbuild.grub2", + Options: options, + } +} + +func (options *GRUB2StageOptions) SetBootFilesystemUUID(u uuid.UUID) { + options.BootFilesystemUUID = u +} + +func (options *GRUB2StageOptions) SetKernelOptions(kernelOptions string) { + options.KernelOptions = kernelOptions +} diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 85aedc51c..d54da5fd8 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -1,18 +1,111 @@ package pipeline +import ( + "encoding/json" + "errors" +) + type Pipeline struct { - Stages []Stage `json:"stages,omitempty"` - Assembler Assembler `json:"assembler"` + BuildPipeline *Pipeline `json:"build,omitempty"` + Stages []*Stage `json:"stages,omitempty"` + Assembler *Assembler `json:"assembler,omitempty"` } type Stage struct { + Name string `json:"name"` + Options StageOptions `json:"options"` +} + +type StageOptions interface { + isStageOptions() +} + +type rawStage struct { + Name string `json:"name"` + Options json.RawMessage `json:"options"` +} + +func (stage *Stage) UnmarshalJSON(data []byte) error { + var rawStage rawStage + err := json.Unmarshal(data, &rawStage) + if err != nil { + return err + } + var options StageOptions + switch rawStage.Name { + case "org.osbuild.dnf": + options = new(DNFStageOptions) + case "org.osbuild.fix-bls": + options = new(FixBLSStageOptions) + case "org.osbuild.FSTab": + options = new(FSTabStageOptions) + case "org.osbuild.grub2": + options = new(GRUB2StageOptions) + case "org.osbuild.locale": + options = new(LocaleStageOptions) + case "org.osbuild.SELinux": + options = new(SELinuxStageOptions) + default: + return errors.New("unexpected stage name") + } + err = json.Unmarshal(rawStage.Options, options) + if err != nil { + return err + } + + stage.Name = rawStage.Name + stage.Options = options + + return nil } type Assembler struct { - Name string `json:"name"` - Options AssemblerTarOptions `json:"options"` + Name string `json:"name"` + Options AssemblerOptions `json:"options"` +} +type AssemblerOptions interface { + isAssemblerOptions() } -type AssemblerTarOptions struct { - Filename string `json:"filename"` +type rawAssembler struct { + Name string `json:"name"` + Options json.RawMessage `json:"options"` +} + +func (assembler *Assembler) UnmarshalJSON(data []byte) error { + var rawAssembler rawAssembler + err := json.Unmarshal(data, &rawAssembler) + if err != nil { + return err + } + var options AssemblerOptions + switch rawAssembler.Name { + case "org.osbuild.tar": + options = new(TarAssemblerOptions) + case "org.osbuild.qcow2": + options = new(QCOW2AssemblerOptions) + default: + return errors.New("unexpected assembler name") + } + err = json.Unmarshal(rawAssembler.Options, options) + if err != nil { + return err + } + + assembler.Name = rawAssembler.Name + assembler.Options = options + + return nil +} + +func (p *Pipeline) SetBuildPipeline(buildPipeline *Pipeline) { + p.BuildPipeline = buildPipeline +} + +func (p *Pipeline) AddStage(stage *Stage) { + p.Stages = append(p.Stages, stage) +} + +func (p *Pipeline) SetAssembler(assembler *Assembler) { + p.Assembler = assembler } diff --git a/internal/pipeline/qcow2_assembler.go b/internal/pipeline/qcow2_assembler.go new file mode 100644 index 000000000..a047b535d --- /dev/null +++ b/internal/pipeline/qcow2_assembler.go @@ -0,0 +1,26 @@ +package pipeline + +import "github.com/google/uuid" + +type QCOW2AssemblerOptions struct { + Filename string `json:"filename"` + RootFilesystemUUDI uuid.UUID `json:"root_fs_uuid"` + Size uint64 `json:"size"` +} + +func (QCOW2AssemblerOptions) isAssemblerOptions() {} + +func NewQCOW2AssemblerOptions(filename string, rootFilesystemUUID uuid.UUID, size uint64) *QCOW2AssemblerOptions { + return &QCOW2AssemblerOptions{ + Filename: filename, + RootFilesystemUUDI: rootFilesystemUUID, + Size: size, + } +} + +func NewQCOW2Assembler(options *QCOW2AssemblerOptions) *Assembler { + return &Assembler{ + Name: "org.osbuild.qcow2", + Options: options, + } +} diff --git a/internal/pipeline/selinux_stage.go b/internal/pipeline/selinux_stage.go new file mode 100644 index 000000000..5f51da577 --- /dev/null +++ b/internal/pipeline/selinux_stage.go @@ -0,0 +1,20 @@ +package pipeline + +type SELinuxStageOptions struct { + FileContexts string `json:"file_contexts"` +} + +func (SELinuxStageOptions) isStageOptions() {} + +func NewSELinuxStageOptions(fileContexts string) *SELinuxStageOptions { + return &SELinuxStageOptions{ + FileContexts: fileContexts, + } +} + +func NewSELinuxStage(options *SELinuxStageOptions) *Stage { + return &Stage{ + Name: "org.osbuild.selinux", + Options: options, + } +} diff --git a/internal/pipeline/tar_assembler.go b/internal/pipeline/tar_assembler.go new file mode 100644 index 000000000..2aaaba556 --- /dev/null +++ b/internal/pipeline/tar_assembler.go @@ -0,0 +1,25 @@ +package pipeline + +type TarAssemblerOptions struct { + Filename string `json:"filename"` + Compression string `json:"compression,omitempty"` +} + +func (TarAssemblerOptions) isAssemblerOptions() {} + +func NewTarAssemblerOptions(filename string) *TarAssemblerOptions { + return &TarAssemblerOptions{ + Filename: filename, + } +} + +func NewTarAssembler(options *TarAssemblerOptions) *Assembler { + return &Assembler{ + Name: "org.osbuild.tar", + Options: options, + } +} + +func (options *TarAssemblerOptions) SetCompression(compression string) { + options.Compression = compression +} diff --git a/internal/target/local_target.go b/internal/target/local_target.go new file mode 100644 index 000000000..a13b4a226 --- /dev/null +++ b/internal/target/local_target.go @@ -0,0 +1,20 @@ +package target + +type LocalTargetOptions struct { + Location string `json:"location"` +} + +func (LocalTargetOptions) isTargetOptions() {} + +func NewLocalTargetOptions(location string) *LocalTargetOptions { + return &LocalTargetOptions{ + Location: location, + } +} + +func NewLocalTarget(options *LocalTargetOptions) *Target { + return &Target{ + Name: "org.osbuild.local", + Options: options, + } +} diff --git a/internal/target/target.go b/internal/target/target.go index 3e69f9dd3..6b546295a 100644 --- a/internal/target/target.go +++ b/internal/target/target.go @@ -1,21 +1,44 @@ package target -import "github.com/google/uuid" +import ( + "encoding/json" + "errors" +) type Target struct { - Name string `json:"name"` - Options Options `json:"options"` + Name string `json:"name"` + Options TargetOptions `json:"options"` } -type Options struct { - Location string `json:"location"` +type TargetOptions interface { + isTargetOptions() } -func New(ComposeID uuid.UUID) *Target { - return &Target{ - Name: "org.osbuild.local", - Options: Options{ - Location: "/var/lib/osbuild-composer/outputs/" + ComposeID.String(), - }, +type rawTarget struct { + Name string `json:"name"` + Options json.RawMessage `json:"options"` +} + +func (target *Target) UnmarshalJSON(data []byte) error { + var rawTarget rawTarget + err := json.Unmarshal(data, &rawTarget) + if err != nil { + return err } + var options TargetOptions + switch rawTarget.Name { + case "org.osbuild.local": + options = new(LocalTargetOptions) + default: + return errors.New("unexpected target name") + } + err = json.Unmarshal(rawTarget.Options, options) + if err != nil { + return err + } + + target.Name = rawTarget.Name + target.Options = options + + return nil } diff --git a/internal/weldr/api.go b/internal/weldr/api.go index d784ca415..22cb03b5e 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -627,7 +627,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, httpRequest *http.Req found := api.store.getBlueprint(cr.BlueprintName, &bp, &changed) // TODO: what to do with changed? if found { - api.store.addCompose(reply.BuildID, bp, cr.ComposeType) + api.store.addCompose(reply.BuildID, &bp, cr.ComposeType) } else { statusResponseError(writer, http.StatusBadRequest, "blueprint does not exist") return diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go index d1a502f6e..acc01fed3 100644 --- a/internal/weldr/api_test.go +++ b/internal/weldr/api_test.go @@ -153,12 +153,4 @@ func TestCompose(t *testing.T) { testRoute(t, api, "POST", "/api/v0/compose", `{"blueprint_name": "test","compose_type": "tar","branch": "master"}`, http.StatusOK, `*`) - - job := <-jobChannel - if job.Pipeline.Assembler.Name != "org.osbuild.tar" { - t.Errorf("Expected tar assembler, got: %s", job.Pipeline.Assembler.Name) - } - if job.Targets[0].Name != "org.osbuild.local" { - t.Errorf("Expected local target, got: %s", job.Targets[0].Name) - } } diff --git a/internal/weldr/store.go b/internal/weldr/store.go index 32551f5bc..9c1a1290a 100644 --- a/internal/weldr/store.go +++ b/internal/weldr/store.go @@ -37,9 +37,9 @@ type blueprintPackage struct { } type compose struct { - Status string `json:"status"` - Pipeline pipeline.Pipeline `json:"pipeline"` - Targets []*target.Target `json:"targets"` + Status string `json:"status"` + Blueprint *blueprint `json:"blueprint"` + Targets []*target.Target `json:"targets"` } func newStore(initialState []byte, stateChannel chan<- []byte, pendingJobs chan<- job.Job, jobUpdates <-chan job.Status) *store { @@ -188,26 +188,22 @@ func (s *store) deleteBlueprintFromWorkspace(name string) { }) } -func (s *store) addCompose(composeID uuid.UUID, bp blueprint, composeType string) { - pipeline := bp.translateToPipeline(composeType) - targets := []*target.Target{target.New(composeID)} +func (s *store) addCompose(composeID uuid.UUID, bp *blueprint, composeType string) { + targets := []*target.Target{ + target.NewLocalTarget(target.NewLocalTargetOptions("/var/lib/osbuild-composer/outputs/" + composeID.String())), + } s.change(func() { - s.Composes[composeID] = compose{"pending", pipeline, targets} + s.Composes[composeID] = compose{"pending", bp, targets} }) s.pendingJobs <- job.Job{ ComposeID: composeID, - Pipeline: pipeline, + Pipeline: bp.translateToPipeline(composeType), Targets: targets, } } -func (b *blueprint) translateToPipeline(outputFormat string) pipeline.Pipeline { - return pipeline.Pipeline{ - Assembler: pipeline.Assembler{ - Name: "org.osbuild.tar", - Options: pipeline.AssemblerTarOptions{ - Filename: "image.tar", - }, - }, - } +func (b *blueprint) translateToPipeline(outputFormat string) *pipeline.Pipeline { + p := &pipeline.Pipeline{} + p.SetAssembler(pipeline.NewTarAssembler(pipeline.NewTarAssemblerOptions("image.tar"))) + return p }