particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

store: drop support for multi-image composes

Browse source 
The store only serves the weldr API, and that hard-codes the assumption
of only one image build per compose all over the place. Move this
assumption into the json serialization handler.

Signed-off-by: Tom Gundersen <teg@jklm.no>
This commit is contained in:
Tom Gundersen 2020-05-12 02:27:46 +02:00
parent e28fca493c
commit fba2af5f51
7 changed files with 141 additions and 168 deletions
Download patch file Download diff file Expand all files Collapse all files

12
internal/store/compose.go
View file

@ -78,8 +78,8 @@ func (ib *ImageBuild) GetLocalTargetOptions() *target.LocalTargetOptions {
// It contains all the information necessary to generate the inputs for the job, as
// well as the job's state.
type Compose struct {
Blueprint *blueprint.Blueprint
ImageBuilds []ImageBuild
Blueprint *blueprint.Blueprint
ImageBuild ImageBuild
}
// DeepCopy creates a copy of the Compose structure
@ -89,12 +89,8 @@ func (c *Compose) DeepCopy() Compose {
bpCopy := *c.Blueprint
newBpPtr = &bpCopy
}
newImageBuilds := []ImageBuild{}
for _, ib := range c.ImageBuilds {
newImageBuilds = append(newImageBuilds, ib.DeepCopy())
}
return Compose{
Blueprint: newBpPtr,
ImageBuilds: newImageBuilds,
Blueprint: newBpPtr,
ImageBuild: c.ImageBuild.DeepCopy(),
}
}

58
internal/store/fixtures.go
View file

@ -62,51 +62,43 @@ func FixtureBase() *Store {
s.composes = map[uuid.UUID]Compose{
uuid.MustParse("30000000-0000-0000-0000-000000000000"): Compose{
Blueprint: &b,
ImageBuilds: []ImageBuild{
{
QueueStatus: common.IBWaiting,
ImageType: imgType,
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
},
ImageBuild: ImageBuild{
QueueStatus: common.IBWaiting,
ImageType: imgType,
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
},
},
uuid.MustParse("30000000-0000-0000-0000-000000000001"): Compose{
Blueprint: &b,
ImageBuilds: []ImageBuild{
{
QueueStatus: common.IBRunning,
ImageType: imgType,
Targets: []*target.Target{localTarget},
JobCreated: date,
JobStarted: date,
},
ImageBuild: ImageBuild{
QueueStatus: common.IBRunning,
ImageType: imgType,
Targets: []*target.Target{localTarget},
JobCreated: date,
JobStarted: date,
},
},
uuid.MustParse("30000000-0000-0000-0000-000000000002"): Compose{
Blueprint: &b,
ImageBuilds: []ImageBuild{
{
QueueStatus: common.IBFinished,
ImageType: imgType,
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
JobStarted: date,
JobFinished: date,
},
ImageBuild: ImageBuild{
QueueStatus: common.IBFinished,
ImageType: imgType,
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
JobStarted: date,
JobFinished: date,
},
},
uuid.MustParse("30000000-0000-0000-0000-000000000003"): Compose{
Blueprint: &b,
ImageBuilds: []ImageBuild{
{
QueueStatus: common.IBFailed,
ImageType: imgType,
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
JobStarted: date,
JobFinished: date,
},
ImageBuild: ImageBuild{
QueueStatus: common.IBFailed,
ImageType: imgType,
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
JobStarted: date,
JobFinished: date,
},
},
}

67
internal/store/json.go
View file

@ -1,6 +1,7 @@
package store
import (
"errors"
"sort"
"time"
@ -95,9 +96,9 @@ func newComposesFromV0(composesStruct composesV0, arch distro.Arch) map[uuid.UUI
composes := make(map[uuid.UUID]Compose)
for composeID, composeStruct := range composesStruct {
c := newComposeFromV0(composeStruct, arch)
if len(c.ImageBuilds) == 0 {
// In case no valid image builds were found, ignore the compose.
c, err := newComposeFromV0(composeStruct, arch)
if err != nil {
// Ignore invalid composes.
continue
}
composes[composeID] = c
@ -106,7 +107,7 @@ func newComposesFromV0(composesStruct composesV0, arch distro.Arch) map[uuid.UUI
return composes
}
func newComposeFromV0(composeStruct composeV0, arch distro.Arch) Compose {
func newComposeFromV0(composeStruct composeV0, arch distro.Arch) (Compose, error) {
c := Compose{
Blueprint: composeStruct.Blueprint,
}
@ -114,8 +115,8 @@ func newComposeFromV0(composeStruct composeV0, arch distro.Arch) Compose {
imgType := imageTypeFromCompatString(imgBuild.ImageType, arch)
if imgType == nil {
// Invalid type strings in serialization format, this may happen
// on upgrades. Ignore the image build.
continue
// on upgrades.
return Compose{}, errors.New("Invalid Image Type string.")
}
ib := ImageBuild{
ID: imgBuild.ID,
@ -129,9 +130,10 @@ func newComposeFromV0(composeStruct composeV0, arch distro.Arch) Compose {
JobID: imgBuild.JobID,
QueueStatus: imgBuild.QueueStatus,
}
c.ImageBuilds = append(c.ImageBuilds, ib)
c.ImageBuild = ib
return c, nil
}
return c
return Compose{}, errors.New("No valid image build found in compose.")
}
func newSourceConfigsFromV0(sourcesStruct sourcesV0) map[string]SourceConfig {
@ -186,15 +188,11 @@ func newStoreFromV0(storeStruct storeV0, arch distro.Arch) *Store {
// (and the compose). The fields are kept so that previously
// succeeded builds still show up correctly.
for composeID, compose := range store.composes {
if len(compose.ImageBuilds) == 0 {
panic("the was a compose with zero image builds, that is forbidden")
}
for imgID, imgBuild := range compose.ImageBuilds {
switch imgBuild.QueueStatus {
case common.IBRunning, common.IBWaiting:
compose.ImageBuilds[imgID].QueueStatus = common.IBFailed
store.composes[composeID] = compose
}
switch compose.ImageBuild.QueueStatus {
case common.IBRunning, common.IBWaiting:
compose.ImageBuild.QueueStatus = common.IBFailed
store.composes[composeID] = compose
}
}
@ -247,28 +245,23 @@ func newComposesV0(composes map[uuid.UUID]Compose) composesV0 {
c := composeV0{
Blueprint: compose.Blueprint,
}
if len(compose.ImageBuilds) == 0 {
panic("the was a compose with zero image builds, that is forbidden")
imgType := imageTypeToCompatString(compose.ImageBuild.ImageType)
if imgType == "" {
panic("invalid image type: " + compose.ImageBuild.ImageType.Name())
}
for _, imgBuild := range compose.ImageBuilds {
imgType := imageTypeToCompatString(imgBuild.ImageType)
if imgType == "" {
panic("invalid image type: " + imgBuild.ImageType.Name())
}
ib := imageBuildV0{
ID: imgBuild.ID,
ImageType: imgType,
Manifest: imgBuild.Manifest,
Targets: imgBuild.Targets,
JobCreated: imgBuild.JobCreated,
JobStarted: imgBuild.JobStarted,
JobFinished: imgBuild.JobFinished,
Size: imgBuild.Size,
JobID: imgBuild.JobID,
QueueStatus: imgBuild.QueueStatus,
}
c.ImageBuilds = append(c.ImageBuilds, ib)
ib := imageBuildV0{
ID: compose.ImageBuild.ID,
ImageType: imgType,
Manifest: compose.ImageBuild.Manifest,
Targets: compose.ImageBuild.Targets,
JobCreated: compose.ImageBuild.JobCreated,
JobStarted: compose.ImageBuild.JobStarted,
JobFinished: compose.ImageBuild.JobFinished,
Size: compose.ImageBuild.Size,
JobID: compose.ImageBuild.JobID,
QueueStatus: compose.ImageBuild.QueueStatus,
}
c.ImageBuilds = append(c.ImageBuilds, ib)
composesStruct[composeID] = c
}
return composesStruct

66
internal/store/store.go
View file

@ -342,27 +342,27 @@ func (s *Store) GetAllComposes() map[uuid.UUID]Compose {
return composes
}
func (s *Store) GetImageBuildResult(composeId uuid.UUID, imageBuildId int) (io.ReadCloser, error) {
func (s *Store) GetImageBuildResult(composeId uuid.UUID) (io.ReadCloser, error) {
if s.stateDir == nil {
return ioutil.NopCloser(bytes.NewBuffer([]byte("{}"))), nil
}
return os.Open(s.getImageBuildDirectory(composeId, imageBuildId) + "/result.json")
return os.Open(s.getImageBuildDirectory(composeId) + "/result.json")
}
func (s *Store) GetImageBuildImage(composeId uuid.UUID, imageBuildId int) (io.ReadCloser, int64, error) {
func (s *Store) GetImageBuildImage(composeId uuid.UUID) (io.ReadCloser, int64, error) {
c, ok := s.composes[composeId]
if !ok {
return nil, 0, &NotFoundError{"compose does not exist"}
}
localTargetOptions := c.ImageBuilds[imageBuildId].GetLocalTargetOptions()
localTargetOptions := c.ImageBuild.GetLocalTargetOptions()
if localTargetOptions == nil {
return nil, 0, &NoLocalTargetError{"compose does not have local target"}
}
path := fmt.Sprintf("%s/%s", s.getImageBuildDirectory(composeId, imageBuildId), localTargetOptions.Filename)
path := fmt.Sprintf("%s/%s", s.getImageBuildDirectory(composeId), localTargetOptions.Filename)
f, err := os.Open(path)
@ -384,8 +384,9 @@ func (s *Store) getComposeDirectory(composeID uuid.UUID) string {
return fmt.Sprintf("%s/outputs/%s", *s.stateDir, composeID.String())
}
func (s *Store) getImageBuildDirectory(composeID uuid.UUID, imageBuildID int) string {
return fmt.Sprintf("%s/%d", s.getComposeDirectory(composeID), imageBuildID)
func (s *Store) getImageBuildDirectory(composeID uuid.UUID) string {
// only one image build is supported per compose
return fmt.Sprintf("%s/0", s.getComposeDirectory(composeID))
}
func (s *Store) PushCompose(composeID uuid.UUID, manifest *osbuild.Manifest, imageType distro.ImageType, bp *blueprint.Blueprint, size uint64, targets []*target.Target, jobId uuid.UUID) error {
@ -398,7 +399,7 @@ func (s *Store) PushCompose(composeID uuid.UUID, manifest *osbuild.Manifest, ima
}
if s.stateDir != nil {
outputDir := s.getImageBuildDirectory(composeID, 0)
outputDir := s.getImageBuildDirectory(composeID)
err := os.MkdirAll(outputDir, 0755)
if err != nil {
@ -410,15 +411,13 @@ func (s *Store) PushCompose(composeID uuid.UUID, manifest *osbuild.Manifest, ima
_ = s.change(func() error {
s.composes[composeID] = Compose{
Blueprint: bp,
ImageBuilds: []ImageBuild{
{
Manifest: manifest,
ImageType: imageType,
Targets: targets,
JobCreated: time.Now(),
Size: size,
JobID: jobId,
},
ImageBuild: ImageBuild{
Manifest: manifest,
ImageType: imageType,
Targets: targets,
JobCreated: time.Now(),
Size: size,
JobID: jobId,
},
}
return nil
@ -435,14 +434,14 @@ func (s *Store) PushTestCompose(composeID uuid.UUID, manifest *osbuild.Manifest,
}
if s.stateDir != nil {
outputDir := s.getImageBuildDirectory(composeID, 0)
outputDir := s.getImageBuildDirectory(composeID)
err := os.MkdirAll(outputDir, 0755)
if err != nil {
return fmt.Errorf("cannot create output directory for job %v: %#v", composeID, err)
}
f, err := os.Create(s.getImageBuildDirectory(composeID, 0) + "/result.json")
f, err := os.Create(s.getImageBuildDirectory(composeID) + "/result.json")
if err != nil {
return fmt.Errorf("cannot open result.json for job %v: %#v", composeID, err)
}
@ -463,17 +462,14 @@ func (s *Store) PushTestCompose(composeID uuid.UUID, manifest *osbuild.Manifest,
_ = s.change(func() error {
s.composes[composeID] = Compose{
Blueprint: bp,
ImageBuilds: []ImageBuild{
{
QueueStatus: status,
Manifest: manifest,
ImageType: imageType,
Targets: targets,
JobCreated: time.Now(),
JobStarted: time.Now(),
JobFinished: time.Now(),
Size: size,
},
ImageBuild: ImageBuild{
QueueStatus: status,
Manifest: manifest,
ImageType: imageType,
Targets: targets,
JobCreated: time.Now(),
JobStarted: time.Now(),
Size: size,
},
}
return nil
@ -505,17 +501,21 @@ func (s *Store) DeleteCompose(id uuid.UUID) error {
}
func (s *Store) AddImageToImageUpload(composeID uuid.UUID, imageBuildID int, reader io.Reader) error {
if imageBuildID != 0 {
return &NotFoundError{"image build does not exist"}
}
currentCompose, exists := s.composes[composeID]
if !exists {
return &NotFoundError{"compose does not exist"}
}
localTargetOptions := currentCompose.ImageBuilds[imageBuildID].GetLocalTargetOptions()
localTargetOptions := currentCompose.ImageBuild.GetLocalTargetOptions()
if localTargetOptions == nil {
return &NoLocalTargetError{fmt.Sprintf("image upload requested for compse %s and image build %d but it has no local target", composeID.String(), imageBuildID)}
return &NoLocalTargetError{fmt.Sprintf("image upload requested for compse %s, but it has no local target", composeID.String())}
}
path := fmt.Sprintf("%s/%s", s.getImageBuildDirectory(composeID, imageBuildID), localTargetOptions.Filename)
path := fmt.Sprintf("%s/%s", s.getImageBuildDirectory(composeID), localTargetOptions.Filename)
f, err := os.Create(path)
if err != nil {

32
internal/weldr/api.go
View file

@ -146,17 +146,13 @@ func (api *API) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
// queued, started, and finished. Assumes that there's only one image in the
// compose. Returns CWaiting on error.
func (api *API) getComposeState(compose store.Compose) (state common.ComposeState, queued, started, finished time.Time) {
if len(compose.ImageBuilds) == 0 {
return
}
jobId := compose.ImageBuilds[0].JobID
jobId := compose.ImageBuild.JobID
// backwards compatibility: composes that were around before splitting
// the job queue from the store still contain their valid status and
// times. Return those here as a fallback.
if jobId == uuid.Nil {
switch compose.ImageBuilds[0].QueueStatus {
switch compose.ImageBuild.QueueStatus {
case common.IBWaiting:
state = common.CWaiting
case common.IBRunning:
@ -166,9 +162,9 @@ func (api *API) getComposeState(compose store.Compose) (state common.ComposeStat
case common.IBFailed:
state = common.CFailed
}
queued = compose.ImageBuilds[0].JobCreated
started = compose.ImageBuilds[0].JobStarted
finished = compose.ImageBuilds[0].JobFinished
queued = compose.ImageBuild.JobCreated
started = compose.ImageBuild.JobStarted
finished = compose.ImageBuild.JobFinished
return
}
@ -1813,7 +1809,7 @@ func (api *API) composeStatusHandler(writer http.ResponseWriter, request *http.R
continue
} else if filterStatus != "" && state.ToString() != filterStatus {
continue
} else if filterImageType != "" && compose.ImageBuilds[0].ImageType.Name() != filterImageType {
} else if filterImageType != "" && compose.ImageBuild.ImageType.Name() != filterImageType {
continue
}
filteredUUIDs = append(filteredUUIDs, id)
@ -1884,12 +1880,12 @@ func (api *API) composeInfoHandler(writer http.ResponseWriter, request *http.Req
// Weldr API assumes only one image build per compose, that's why only the
// 1st build is considered
state, _, _, _ := api.getComposeState(compose)
reply.ComposeType = compose.ImageBuilds[0].ImageType.Name()
reply.ComposeType = compose.ImageBuild.ImageType.Name()
reply.QueueStatus = state.ToString()
reply.ImageSize = compose.ImageBuilds[0].Size
reply.ImageSize = compose.ImageBuild.Size
if isRequestVersionAtLeast(params, 1) {
reply.Uploads = targetsToUploadResponses(compose.ImageBuilds[0].Targets)
reply.Uploads = targetsToUploadResponses(compose.ImageBuild.Targets)
}
err = json.NewEncoder(writer).Encode(reply)
@ -1932,10 +1928,10 @@ func (api *API) composeImageHandler(writer http.ResponseWriter, request *http.Re
return
}
imageName := compose.ImageBuilds[0].ImageType.Filename()
imageMime := compose.ImageBuilds[0].ImageType.MIMEType()
imageName := compose.ImageBuild.ImageType.Filename()
imageMime := compose.ImageBuild.ImageType.MIMEType()
reader, fileSize, err := api.store.GetImageBuildImage(uuid, 0)
reader, fileSize, err := api.store.GetImageBuildImage(uuid)
// TODO: this might return misleading error
if err != nil {
@ -1991,7 +1987,7 @@ func (api *API) composeLogsHandler(writer http.ResponseWriter, request *http.Req
return
}
resultReader, err := api.store.GetImageBuildResult(id, 0)
resultReader, err := api.store.GetImageBuildResult(id)
if err != nil {
errors := responseError{
@ -2084,7 +2080,7 @@ func (api *API) composeLogHandler(writer http.ResponseWriter, request *http.Requ
return
}
resultReader, err := api.store.GetImageBuildResult(id, 0)
resultReader, err := api.store.GetImageBuildResult(id)
if err != nil {
errors := responseError{

68
internal/weldr/api_test.go
View file

@ -446,17 +446,15 @@ func TestCompose(t *testing.T) {
Groups: []blueprint.Group{},
Customizations: nil,
},
ImageBuilds: []store.ImageBuild{
{
QueueStatus: common.IBWaiting,
ImageType: imgType,
Targets: []*target.Target{
{
// skip Uuid and Created fields - they are ignored
Name: "org.osbuild.local",
Options: &target.LocalTargetOptions{
Filename: "test.img",
},
ImageBuild: store.ImageBuild{
QueueStatus: common.IBWaiting,
ImageType: imgType,
Targets: []*target.Target{
{
// skip Uuid and Created fields - they are ignored
Name: "org.osbuild.local",
Options: &target.LocalTargetOptions{
Filename: "test.img",
},
},
},
@ -471,30 +469,28 @@ func TestCompose(t *testing.T) {
Groups: []blueprint.Group{},
Customizations: nil,
},
ImageBuilds: []store.ImageBuild{
{
QueueStatus: common.IBWaiting,
ImageType: imgType,
Targets: []*target.Target{
{
Name: "org.osbuild.aws",
Status: common.IBWaiting,
ImageName: "test_upload",
Options: &target.AWSTargetOptions{
Filename: "test.img",
Region: "frankfurt",
AccessKeyID: "accesskey",
SecretAccessKey: "secretkey",
Bucket: "clay",
Key: "imagekey",
},
ImageBuild: store.ImageBuild{
QueueStatus: common.IBWaiting,
ImageType: imgType,
Targets: []*target.Target{
{
Name: "org.osbuild.aws",
Status: common.IBWaiting,
ImageName: "test_upload",
Options: &target.AWSTargetOptions{
Filename: "test.img",
Region: "frankfurt",
AccessKeyID: "accesskey",
SecretAccessKey: "secretkey",
Bucket: "clay",
Key: "imagekey",
},
{
// skip Uuid and Created fields - they are ignored
Name: "org.osbuild.local",
Options: &target.LocalTargetOptions{
Filename: "test.img",
},
},
{
// skip Uuid and Created fields - they are ignored
Name: "org.osbuild.local",
Options: &target.LocalTargetOptions{
Filename: "test.img",
},
},
},
@ -535,9 +531,9 @@ func TestCompose(t *testing.T) {
break
}
require.NotNilf(t, composeStruct.ImageBuilds[0].Manifest, "%s: the compose in the store did not contain a blueprint", c.Path)
require.NotNilf(t, composeStruct.ImageBuild.Manifest, "%s: the compose in the store did not contain a blueprint", c.Path)
// TODO: find some (reasonable) way to verify the contents of the pipeline
composeStruct.ImageBuilds[0].Manifest = nil
composeStruct.ImageBuild.Manifest = nil
if diff := cmp.Diff(composeStruct, *c.ExpectedCompose, test.IgnoreDates(), test.IgnoreUuids(), test.Ignore("Targets.Options.Location"), test.CompareImageTypes()); diff != "" {
t.Errorf("%s: compose in store isn't the same as expected, diff:\n%s", c.Path, diff)

6
internal/weldr/compose.go
View file

@ -28,10 +28,10 @@ func composeToComposeEntry(id uuid.UUID, compose store.Compose, state common.Com
composeEntry.ID = id
composeEntry.Blueprint = compose.Blueprint.Name
composeEntry.Version = compose.Blueprint.Version
composeEntry.ComposeType = compose.ImageBuilds[0].ImageType.Name()
composeEntry.ComposeType = compose.ImageBuild.ImageType.Name()
if includeUploads {
composeEntry.Uploads = targetsToUploadResponses(compose.ImageBuilds[0].Targets)
composeEntry.Uploads = targetsToUploadResponses(compose.ImageBuild.Targets)
}
switch state {
@ -46,7 +46,7 @@ func composeToComposeEntry(id uuid.UUID, compose store.Compose, state common.Com
case common.CFinished:
composeEntry.QueueStatus = common.IBFinished
composeEntry.ImageSize = compose.ImageBuilds[0].Size
composeEntry.ImageSize = compose.ImageBuild.Size
composeEntry.JobCreated = float64(queued.UnixNano()) / 1000000000
composeEntry.JobStarted = float64(started.UnixNano()) / 1000000000
composeEntry.JobFinished = float64(finished.UnixNano()) / 1000000000