From ab6b6da76c89694fb22f134eb0e231c57563364b Mon Sep 17 00:00:00 2001 From: Martin Sehnoutka Date: Mon, 10 Feb 2020 09:43:49 +0100 Subject: [PATCH] Weldr API: use the new compose/store API The compose now contains multiple image builds, but Weldr API does not support this feature. Use the first image build every time. Also start using the new types instead of plain strings. --- internal/mocks/rpmmd/fixtures.go | 91 +++++++++++++++---------- internal/weldr/api.go | 61 ++++++++++------- internal/weldr/api_test.go | 113 +++++++++++++++++-------------- internal/weldr/compose.go | 65 +++++++++--------- internal/weldr/upload.go | 15 ++-- 5 files changed, 193 insertions(+), 152 deletions(-) diff --git a/internal/mocks/rpmmd/fixtures.go b/internal/mocks/rpmmd/fixtures.go index 229743fc1..11976cccd 100644 --- a/internal/mocks/rpmmd/fixtures.go +++ b/internal/mocks/rpmmd/fixtures.go @@ -2,12 +2,15 @@ package rpmmd_mock import ( "fmt" + "github.com/osbuild/osbuild-composer/internal/common" + "github.com/osbuild/osbuild-composer/internal/compose" + "github.com/osbuild/osbuild-composer/internal/distro" "sort" "time" "github.com/google/uuid" "github.com/osbuild/osbuild-composer/internal/blueprint" - test_distro "github.com/osbuild/osbuild-composer/internal/distro/test" + test_distro "github.com/osbuild/osbuild-composer/internal/distro/fedoratest" "github.com/osbuild/osbuild-composer/internal/rpmmd" "github.com/osbuild/osbuild-composer/internal/store" "github.com/osbuild/osbuild-composer/internal/target" @@ -71,7 +74,7 @@ func createBaseStoreFixture() *store.Store { Name: "org.osbuild.local", ImageName: "localimage", Created: date, - Status: "WAITING", + Status: common.IBWaiting, Options: &target.LocalTargetOptions{ Location: "/tmp/localimage", }, @@ -82,7 +85,7 @@ func createBaseStoreFixture() *store.Store { Name: "org.osbuild.aws", ImageName: "awsimage", Created: date, - Status: "WAITING", + Status: common.IBWaiting, Options: &target.AWSTargetOptions{ Region: "frankfurt", AccessKeyID: "accesskey", @@ -93,42 +96,59 @@ func createBaseStoreFixture() *store.Store { } d := test_distro.New() - s := store.New(nil, d) + r := distro.NewRegistry([]string{"."}) + s := store.New(nil, d, *r) s.Blueprints[bName] = b - s.Composes = map[uuid.UUID]store.Compose{ - uuid.MustParse("30000000-0000-0000-0000-000000000000"): store.Compose{ - QueueStatus: "WAITING", - Blueprint: &b, - OutputType: "test_output", - Targets: []*target.Target{localTarget, awsTarget}, - JobCreated: date, + s.Composes = map[uuid.UUID]compose.Compose{ + uuid.MustParse("30000000-0000-0000-0000-000000000000"): compose.Compose{ + Blueprint: &b, + ImageBuilds: []compose.ImageBuild{ + { + QueueStatus: common.IBWaiting, + ImageType: common.Qcow2Generic, + Targets: []*target.Target{localTarget, awsTarget}, + JobCreated: date, + }, + }, }, - uuid.MustParse("30000000-0000-0000-0000-000000000001"): store.Compose{ - QueueStatus: "RUNNING", - Blueprint: &b, - OutputType: "test_output", - Targets: []*target.Target{localTarget}, - JobCreated: date, - JobStarted: date, + uuid.MustParse("30000000-0000-0000-0000-000000000001"): compose.Compose{ + Blueprint: &b, + ImageBuilds: []compose.ImageBuild{ + { + QueueStatus: common.IBRunning, + ImageType: common.Qcow2Generic, + Targets: []*target.Target{localTarget}, + JobCreated: date, + JobStarted: date, + }, + }, }, - uuid.MustParse("30000000-0000-0000-0000-000000000002"): store.Compose{ - QueueStatus: "FINISHED", - Blueprint: &b, - OutputType: "test_output", - Targets: []*target.Target{localTarget, awsTarget}, - JobCreated: date, - JobStarted: date, - JobFinished: date, + uuid.MustParse("30000000-0000-0000-0000-000000000002"): compose.Compose{ + Blueprint: &b, + ImageBuilds: []compose.ImageBuild{ + { + QueueStatus: common.IBFinished, + ImageType: common.Qcow2Generic, + Targets: []*target.Target{localTarget, awsTarget}, + JobCreated: date, + JobStarted: date, + JobFinished: date, + }, + }, }, - uuid.MustParse("30000000-0000-0000-0000-000000000003"): store.Compose{ - QueueStatus: "FAILED", - Blueprint: &b, - OutputType: "test_output", - Targets: []*target.Target{localTarget, awsTarget}, - JobCreated: date, - JobStarted: date, - JobFinished: date, + uuid.MustParse("30000000-0000-0000-0000-000000000003"): compose.Compose{ + Blueprint: &b, + ImageBuilds: []compose.ImageBuild{ + { + QueueStatus: common.IBFailed, + ImageType: common.Qcow2Generic, + Targets: []*target.Target{localTarget, awsTarget}, + JobCreated: date, + JobStarted: date, + JobFinished: date, + }, + }, }, } @@ -166,7 +186,8 @@ func createStoreWithoutComposesFixture() *store.Store { } d := test_distro.New() - s := store.New(nil, d) + r := distro.NewRegistry([]string{"."}) + s := store.New(nil, d, *r) s.Blueprints[bName] = b diff --git a/internal/weldr/api.go b/internal/weldr/api.go index 746ee45de..6fdbf02ef 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -4,7 +4,7 @@ import ( "archive/tar" "bytes" "encoding/json" - "errors" + errors_package "errors" "fmt" "io" "log" @@ -356,7 +356,7 @@ func (api *API) sourceNewHandler(writer http.ResponseWriter, request *http.Reque } else if contentType[0] == "text/x-toml" { _, err = toml.DecodeReader(request.Body, &source) } else { - err = errors.New("blueprint must be in json or toml format") + err = errors_package.New("blueprint must be in json or toml format") } if err != nil { @@ -1103,7 +1103,7 @@ func (api *API) blueprintsNewHandler(writer http.ResponseWriter, request *http.R } else if contentType[0] == "text/x-toml" { _, err = toml.DecodeReader(request.Body, &blueprint) } else { - err = errors.New("blueprint must be in json or toml format") + err = errors_package.New("blueprint must be in json or toml format") } if err != nil { @@ -1143,7 +1143,7 @@ func (api *API) blueprintsWorkspaceHandler(writer http.ResponseWriter, request * } else if contentType[0] == "text/x-toml" { _, err = toml.DecodeReader(request.Body, &blueprint) } else { - err = errors.New("blueprint must be in json or toml format") + err = errors_package.New("blueprint must be in json or toml format") } if err != nil { @@ -1211,6 +1211,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request BuildID uuid.UUID `json:"build_id"` Status bool `json:"status"` } + const MegaByte = 1024 * 1024 contentType := request.Header["Content-Type"] if len(contentType) != 1 || contentType[0] != "application/json" { @@ -1256,6 +1257,13 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request bp := api.store.GetBlueprintCommitted(cr.BlueprintName) + size := cr.Size + + // Microsoft Azure requires vhd images to be rounded up to the nearest MB + if cr.ComposeType == "vhd" && size%MegaByte != 0 { + size = (size/MegaByte + 1) * MegaByte + } + if bp != nil { _, checksums, err := api.depsolveBlueprint(bp, true) if err != nil { @@ -1267,7 +1275,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request return } - err = api.store.PushCompose(reply.BuildID, bp, checksums, api.arch, cr.ComposeType, cr.Size, uploadTarget) + err = api.store.PushCompose(reply.BuildID, bp, checksums, api.arch, cr.ComposeType, size, uploadTarget) // TODO: we should probably do some kind of blueprint validation in future // for now, let's just 500 and bail out @@ -1386,10 +1394,10 @@ func (api *API) composeQueueHandler(writer http.ResponseWriter, request *http.Re composes := api.store.GetAllComposes() for id, compose := range composes { - switch compose.QueueStatus { - case "WAITING": + switch compose.GetState() { + case common.CWaiting: reply.New = append(reply.New, composeToComposeEntry(id, compose, isRequestVersionAtLeast(params, 1))) - case "RUNNING": + case common.CRunning: reply.Run = append(reply.Run, composeToComposeEntry(id, compose, isRequestVersionAtLeast(params, 1))) } } @@ -1428,7 +1436,6 @@ func (api *API) composeStatusHandler(writer http.ResponseWriter, request *http.R } } composes := api.store.GetAllComposes() - reply.UUIDs = composesToComposeEntries(composes, uuids, isRequestVersionAtLeast(params, 1)) json.NewEncoder(writer).Encode(reply) @@ -1482,14 +1489,16 @@ func (api *API) composeInfoHandler(writer http.ResponseWriter, request *http.Req reply.Deps = Dependencies{ Packages: make([]map[string]interface{}, 0), } - reply.ComposeType = compose.OutputType - reply.QueueStatus = compose.QueueStatus - if compose.Image != nil { - reply.ImageSize = compose.Size + // Weldr API assumes only one image build per compose, that's why only the + // 1st build is considered + reply.ComposeType, _ = compose.ImageBuilds[0].ImageType.ToCompatString() + reply.QueueStatus = compose.GetState().ToString() + if compose.ImageBuilds[0].Image != nil { + reply.ImageSize = compose.ImageBuilds[0].Size } if isRequestVersionAtLeast(params, 1) { - reply.Uploads = TargetsToUploadResponses(compose.Targets) + reply.Uploads = TargetsToUploadResponses(compose.ImageBuilds[0].Targets) } json.NewEncoder(writer).Encode(reply) @@ -1521,16 +1530,16 @@ func (api *API) composeImageHandler(writer http.ResponseWriter, request *http.Re return } - if compose.QueueStatus != "FINISHED" { + if compose.GetState() != common.CFinished { errors := responseError{ ID: "BuildInWrongState", - Msg: fmt.Sprintf("Build %s is in wrong state: %s", uuidString, compose.QueueStatus), + Msg: fmt.Sprintf("Build %s is in wrong state: %s", uuidString, compose.GetState().ToString()), } statusResponseError(writer, http.StatusBadRequest, errors) return } - if compose.Image == nil { + if compose.ImageBuilds[0].Image == nil { errors := responseError{ ID: "BuildMissingFile", Msg: fmt.Sprintf("Compose %s doesn't have an image assigned", uuidString), @@ -1539,9 +1548,9 @@ func (api *API) composeImageHandler(writer http.ResponseWriter, request *http.Re return } - imageName := filepath.Base(compose.Image.Path) + imageName := filepath.Base(compose.ImageBuilds[0].Image.Path) - file, err := os.Open(compose.Image.Path) + file, err := os.Open(compose.ImageBuilds[0].Image.Path) if err != nil { errors := responseError{ ID: "BuildMissingFile", @@ -1552,8 +1561,8 @@ func (api *API) composeImageHandler(writer http.ResponseWriter, request *http.Re } writer.Header().Set("Content-Disposition", "attachment; filename="+uuid.String()+"-"+imageName) - writer.Header().Set("Content-Type", compose.Image.Mime) - writer.Header().Set("Content-Length", fmt.Sprintf("%d", compose.Image.Size)) + writer.Header().Set("Content-Type", compose.ImageBuilds[0].Image.Mime) + writer.Header().Set("Content-Length", fmt.Sprintf("%d", compose.ImageBuilds[0].Image.Size)) io.Copy(writer, file) } @@ -1584,7 +1593,7 @@ func (api *API) composeLogsHandler(writer http.ResponseWriter, request *http.Req return } - if compose.QueueStatus != "FINISHED" && compose.QueueStatus != "FAILED" { + if compose.GetState() != common.CFinished && compose.GetState() != common.CFailed { errors := responseError{ ID: "BuildInWrongState", Msg: fmt.Sprintf("Build %s not in FINISHED or FAILED state.", uuidString), @@ -1664,7 +1673,7 @@ func (api *API) composeLogHandler(writer http.ResponseWriter, request *http.Requ return } - if compose.QueueStatus == "WAITING" { + if compose.GetState() == common.CWaiting { errors := responseError{ ID: "BuildInWrongState", Msg: fmt.Sprintf("Build %s has not started yet. No logs to view.", uuidString), @@ -1673,7 +1682,7 @@ func (api *API) composeLogHandler(writer http.ResponseWriter, request *http.Requ return } - if compose.QueueStatus == "RUNNING" { + if compose.GetState() == common.CRunning { fmt.Fprintf(writer, "Running...\n") return } @@ -1717,7 +1726,7 @@ func (api *API) composeFinishedHandler(writer http.ResponseWriter, request *http composes := api.store.GetAllComposes() for _, entry := range composesToComposeEntries(composes, nil, isRequestVersionAtLeast(params, 1)) { switch entry.QueueStatus { - case "FINISHED": + case common.IBFinished: reply.Finished = append(reply.Finished, entry) } } @@ -1737,7 +1746,7 @@ func (api *API) composeFailedHandler(writer http.ResponseWriter, request *http.R composes := api.store.GetAllComposes() for _, entry := range composesToComposeEntries(composes, nil, isRequestVersionAtLeast(params, 1)) { switch entry.QueueStatus { - case "FAILED": + case common.IBFailed: reply.Failed = append(reply.Failed, entry) } } diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go index ad22e3d9b..b8dd065cb 100644 --- a/internal/weldr/api_test.go +++ b/internal/weldr/api_test.go @@ -3,6 +3,9 @@ package weldr_test import ( "archive/tar" "bytes" + "github.com/osbuild/osbuild-composer/internal/common" + "github.com/osbuild/osbuild-composer/internal/compose" + "github.com/osbuild/osbuild-composer/internal/target" "io" "math/rand" "net/http" @@ -13,11 +16,9 @@ import ( "time" "github.com/osbuild/osbuild-composer/internal/blueprint" - _ "github.com/osbuild/osbuild-composer/internal/distro/test" - test_distro "github.com/osbuild/osbuild-composer/internal/distro/test" + test_distro "github.com/osbuild/osbuild-composer/internal/distro/fedoratest" rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd" "github.com/osbuild/osbuild-composer/internal/store" - "github.com/osbuild/osbuild-composer/internal/target" "github.com/osbuild/osbuild-composer/internal/test" "github.com/osbuild/osbuild-composer/internal/weldr" @@ -31,7 +32,7 @@ func createWeldrAPI(fixtureGenerator rpmmd_mock.FixtureGenerator) (*weldr.API, * rpm := rpmmd_mock.NewRPMMDMock(fixture) d := test_distro.New() - return weldr.New(rpm, "test_arch", d, nil, fixture.Store), fixture.Store + return weldr.New(rpm, "x86_64", d, nil, fixture.Store), fixture.Store } func TestBasic(t *testing.T) { @@ -47,8 +48,8 @@ func TestBasic(t *testing.T) { {"/api/v0/projects/source/info", http.StatusNotFound, ``}, {"/api/v0/projects/source/info/", http.StatusNotFound, `{"errors":[{"code":404,"id":"HTTPError","msg":"Not Found"}],"status":false}`}, {"/api/v0/projects/source/info/foo", http.StatusOK, `{"errors":[{"id":"UnknownSource","msg":"foo is not a valid source"}],"sources":{}}`}, - {"/api/v0/projects/source/info/test-id", http.StatusOK, `{"sources":{"test-id":{"name":"test-id","type":"yum-baseurl","url":"http://example.com/test/os/test_arch","check_gpg":true,"check_ssl":true,"system":true}},"errors":[]}`}, - {"/api/v0/projects/source/info/*", http.StatusOK, `{"sources":{"test-id":{"name":"test-id","type":"yum-baseurl","url":"http://example.com/test/os/test_arch","check_gpg":true,"check_ssl":true,"system":true}},"errors":[]}`}, + {"/api/v0/projects/source/info/test-id", http.StatusOK, `{"sources":{"test-id":{"name":"test-id","type":"yum-baseurl","url":"http://example.com/test/os/x86_64","check_gpg":true,"check_ssl":true,"system":true}},"errors":[]}`}, + {"/api/v0/projects/source/info/*", http.StatusOK, `{"sources":{"test-id":{"name":"test-id","type":"yum-baseurl","url":"http://example.com/test/os/x86_64","check_gpg":true,"check_ssl":true,"system":true}},"errors":[]}`}, {"/api/v0/blueprints/list", http.StatusOK, `{"total":1,"offset":0,"limit":1,"blueprints":["test"]}`}, {"/api/v0/blueprints/info/", http.StatusNotFound, `{"errors":[{"code":404,"id":"HTTPError","msg":"Not Found"}],"status":false}`}, @@ -293,8 +294,7 @@ func TestBlueprintsDepsolve(t *testing.T) { } func TestCompose(t *testing.T) { - expectedComposeLocal := &store.Compose{ - QueueStatus: "WAITING", + expectedComposeLocal := &compose.Compose{ Blueprint: &blueprint.Blueprint{ Name: "test", Version: "0.0.0", @@ -303,31 +303,40 @@ func TestCompose(t *testing.T) { Groups: []blueprint.Group{}, Customizations: nil, }, - OutputType: "test_output", - Targets: []*target.Target{}, - } - expectedComposeLocalAndAws := &store.Compose{ - QueueStatus: "WAITING", - Blueprint: &blueprint.Blueprint{ - Name: "test", - Version: "0.0.0", - Packages: []blueprint.Package{}, - Modules: []blueprint.Package{}, - Groups: []blueprint.Group{}, - Customizations: nil, - }, - OutputType: "test_output", - Targets: []*target.Target{ + ImageBuilds: []compose.ImageBuild{ { - Name: "org.osbuild.aws", - Status: "WAITING", - ImageName: "test_upload", - Options: &target.AWSTargetOptions{ - Region: "frankfurt", - AccessKeyID: "accesskey", - SecretAccessKey: "secretkey", - Bucket: "clay", - Key: "imagekey", + QueueStatus: common.IBWaiting, + ImageType: common.Qcow2Generic, + Targets: []*target.Target{}, + }, + }, + } + expectedComposeLocalAndAws := &compose.Compose{ + Blueprint: &blueprint.Blueprint{ + Name: "test", + Version: "0.0.0", + Packages: []blueprint.Package{}, + Modules: []blueprint.Package{}, + Groups: []blueprint.Group{}, + Customizations: nil, + }, + ImageBuilds: []compose.ImageBuild{ + { + QueueStatus: common.IBWaiting, + ImageType: common.Qcow2Generic, + Targets: []*target.Target{ + { + Name: "org.osbuild.aws", + Status: common.IBWaiting, + ImageName: "test_upload", + Options: &target.AWSTargetOptions{ + Region: "frankfurt", + AccessKeyID: "accesskey", + SecretAccessKey: "secretkey", + Bucket: "clay", + Key: "imagekey", + }, + }, }, }, }, @@ -340,12 +349,12 @@ func TestCompose(t *testing.T) { Body string ExpectedStatus int ExpectedJSON string - ExpectedCompose *store.Compose + ExpectedCompose *compose.Compose IgnoreFields []string }{ - {true, "POST", "/api/v0/compose", `{"blueprint_name": "http-server","compose_type": "test_output","branch": "master"}`, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownBlueprint","msg":"Unknown blueprint name: http-server"}]}`, nil, []string{"build_id"}}, - {false, "POST", "/api/v0/compose", `{"blueprint_name": "test","compose_type": "test_output","branch": "master"}`, http.StatusOK, `{"status": true}`, expectedComposeLocal, []string{"build_id"}}, - {false, "POST", "/api/v1/compose", `{"blueprint_name": "test","compose_type":"test_output","branch":"master","upload":{"image_name":"test_upload","provider":"aws","settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}}`, http.StatusOK, `{"status": true}`, expectedComposeLocalAndAws, []string{"build_id"}}, + {true, "POST", "/api/v0/compose", `{"blueprint_name": "http-server","compose_type": "qcow2","branch": "master"}`, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownBlueprint","msg":"Unknown blueprint name: http-server"}]}`, nil, []string{"build_id"}}, + {false, "POST", "/api/v0/compose", `{"blueprint_name": "test","compose_type": "qcow2","branch": "master"}`, http.StatusOK, `{"status": true}`, expectedComposeLocal, []string{"build_id"}}, + {false, "POST", "/api/v1/compose", `{"blueprint_name": "test","compose_type":"qcow2","branch":"master","upload":{"image_name":"test_upload","provider":"aws","settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}}`, http.StatusOK, `{"status": true}`, expectedComposeLocalAndAws, []string{"build_id"}}, } for _, c := range cases { @@ -361,20 +370,20 @@ func TestCompose(t *testing.T) { } // I have no idea how to get the compose in better way - var compose store.Compose + var composeStruct compose.Compose for _, c := range s.Composes { - compose = c + composeStruct = c break } - if compose.Pipeline == nil { + if composeStruct.ImageBuilds[0].Pipeline == nil { t.Fatalf("%s: the compose in the store did not contain a blueprint", c.Path) } else { // TODO: find some (reasonable) way to verify the contents of the pipeline - compose.Pipeline = nil + composeStruct.ImageBuilds[0].Pipeline = nil } - if diff := cmp.Diff(compose, *c.ExpectedCompose, test.IgnoreDates(), test.IgnoreUuids(), test.Ignore("Targets.Options.Location")); diff != "" { + if diff := cmp.Diff(composeStruct, *c.ExpectedCompose, test.IgnoreDates(), test.IgnoreUuids(), test.Ignore("Targets.Options.Location")); diff != "" { t.Errorf("%s: compose in store isn't the same as expected, diff:\n%s", c.Path, diff) } @@ -425,9 +434,9 @@ func TestComposeStatus(t *testing.T) { ExpectedStatus int ExpectedJSON string }{ - {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/30000000-0000-0000-0000-000000000000,30000000-0000-0000-0000-000000000002", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, - {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/*", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000001","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"RUNNING","job_created":1574857140,"job_started":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140},{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, - {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/status/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING","job_created":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/30000000-0000-0000-0000-000000000000,30000000-0000-0000-0000-000000000002", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/*", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000001","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"RUNNING","job_created":1574857140,"job_started":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140},{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/status/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING","job_created":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`}, } if len(os.Getenv("OSBUILD_COMPOSER_TEST_EXTERNAL")) > 0 { @@ -449,8 +458,8 @@ func TestComposeInfo(t *testing.T) { ExpectedStatus int ExpectedJSON string }{ - {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/info/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"id":"30000000-0000-0000-0000-000000000000","config":"","blueprint":{"name":"test","description":"","version":"0.0.0","packages":[],"modules":[],"groups":[]},"commit":"","deps":{"packages":[]},"compose_type":"test_output","queue_status":"WAITING","image_size":0}`}, - {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/info/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"id":"30000000-0000-0000-0000-000000000000","config":"","blueprint":{"name":"test","description":"","version":"0.0.0","packages":[],"modules":[],"groups":[]},"commit":"","deps":{"packages":[]},"compose_type":"test_output","queue_status":"WAITING","image_size":0,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/info/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"id":"30000000-0000-0000-0000-000000000000","config":"","blueprint":{"name":"test","description":"","version":"0.0.0","packages":[],"modules":[],"groups":[]},"commit":"","deps":{"packages":[]},"compose_type":"qcow2","queue_status":"WAITING","image_size":0}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/info/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"id":"30000000-0000-0000-0000-000000000000","config":"","blueprint":{"name":"test","description":"","version":"0.0.0","packages":[],"modules":[],"groups":[]},"commit":"","deps":{"packages":[]},"compose_type":"qcow2","queue_status":"WAITING","image_size":0,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}`}, {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/info/30000000-0000-0000-0000", ``, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownUUID","msg":"30000000-0000-0000-0000 is not a valid build uuid"}]}`}, {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/info/42000000-0000-0000-0000-000000000000", ``, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownUUID","msg":"42000000-0000-0000-0000-000000000000 is not a valid build uuid"}]}`}, } @@ -571,8 +580,8 @@ func TestComposeQueue(t *testing.T) { ExpectedStatus int ExpectedJSON string }{ - {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING"}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"RUNNING"}]}`}, - {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING","uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"RUNNING"}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING"}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"RUNNING"}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING","uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"RUNNING"}]}`}, {rpmmd_mock.NoComposesFixture, "GET", "/api/v0/compose/queue", ``, http.StatusOK, `{"new":[],"run":[]}`}, } @@ -595,8 +604,8 @@ func TestComposeFinished(t *testing.T) { ExpectedStatus int ExpectedJSON string }{ - {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, - {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`}, {rpmmd_mock.NoComposesFixture, "GET", "/api/v0/compose/finished", ``, http.StatusOK, `{"finished":[]}`}, } @@ -619,8 +628,8 @@ func TestComposeFailed(t *testing.T) { ExpectedStatus int ExpectedJSON string }{ - {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, - {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`}, + {rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`}, {rpmmd_mock.NoComposesFixture, "GET", "/api/v0/compose/failed", ``, http.StatusOK, `{"failed":[]}`}, } diff --git a/internal/weldr/compose.go b/internal/weldr/compose.go index a4355bab3..404b846d4 100644 --- a/internal/weldr/compose.go +++ b/internal/weldr/compose.go @@ -2,61 +2,62 @@ package weldr import ( "github.com/google/uuid" - "github.com/osbuild/osbuild-composer/internal/store" + "github.com/osbuild/osbuild-composer/internal/common" + "github.com/osbuild/osbuild-composer/internal/compose" "log" "sort" ) type ComposeEntry struct { - ID uuid.UUID `json:"id"` - Blueprint string `json:"blueprint"` - Version string `json:"version"` - ComposeType string `json:"compose_type"` - ImageSize uint64 `json:"image_size"` - QueueStatus string `json:"queue_status"` - JobCreated float64 `json:"job_created"` - JobStarted float64 `json:"job_started,omitempty"` - JobFinished float64 `json:"job_finished,omitempty"` - Uploads []UploadResponse `json:"uploads,omitempty"` + ID uuid.UUID `json:"id"` + Blueprint string `json:"blueprint"` + Version string `json:"version"` + ComposeType common.ImageType `json:"compose_type"` + ImageSize uint64 `json:"image_size"` + QueueStatus common.ImageBuildState `json:"queue_status"` + JobCreated float64 `json:"job_created"` + JobStarted float64 `json:"job_started,omitempty"` + JobFinished float64 `json:"job_finished,omitempty"` + Uploads []UploadResponse `json:"uploads,omitempty"` } -func composeToComposeEntry(id uuid.UUID, compose store.Compose, includeUploads bool) *ComposeEntry { +func composeToComposeEntry(id uuid.UUID, compose compose.Compose, includeUploads bool) *ComposeEntry { var composeEntry ComposeEntry composeEntry.ID = id composeEntry.Blueprint = compose.Blueprint.Name composeEntry.Version = compose.Blueprint.Version - composeEntry.ComposeType = compose.OutputType - composeEntry.QueueStatus = compose.QueueStatus + composeEntry.ComposeType = compose.ImageBuilds[0].ImageType + composeEntry.QueueStatus = compose.ImageBuilds[0].QueueStatus if includeUploads { - composeEntry.Uploads = TargetsToUploadResponses(compose.Targets) + composeEntry.Uploads = TargetsToUploadResponses(compose.ImageBuilds[0].Targets) } - switch compose.QueueStatus { - case "WAITING": - composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000 + switch compose.ImageBuilds[0].QueueStatus { + case common.IBWaiting: + composeEntry.JobCreated = float64(compose.ImageBuilds[0].JobCreated.UnixNano()) / 1000000000 - case "RUNNING": - composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000 - composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000 + case common.IBRunning: + composeEntry.JobCreated = float64(compose.ImageBuilds[0].JobCreated.UnixNano()) / 1000000000 + composeEntry.JobStarted = float64(compose.ImageBuilds[0].JobStarted.UnixNano()) / 1000000000 - case "FINISHED": - if compose.Image != nil { - composeEntry.ImageSize = compose.Size + case common.IBFinished: + if compose.ImageBuilds[0].Image != nil { + composeEntry.ImageSize = compose.ImageBuilds[0].Size } else { log.Printf("finished compose with id %s has nil image\n", id.String()) composeEntry.ImageSize = 0 } - composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000 - composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000 - composeEntry.JobFinished = float64(compose.JobFinished.UnixNano()) / 1000000000 + composeEntry.JobCreated = float64(compose.ImageBuilds[0].JobCreated.UnixNano()) / 1000000000 + composeEntry.JobStarted = float64(compose.ImageBuilds[0].JobStarted.UnixNano()) / 1000000000 + composeEntry.JobFinished = float64(compose.ImageBuilds[0].JobFinished.UnixNano()) / 1000000000 - case "FAILED": - composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000 - composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000 - composeEntry.JobFinished = float64(compose.JobFinished.UnixNano()) / 1000000000 + case common.IBFailed: + composeEntry.JobCreated = float64(compose.ImageBuilds[0].JobCreated.UnixNano()) / 1000000000 + composeEntry.JobStarted = float64(compose.ImageBuilds[0].JobStarted.UnixNano()) / 1000000000 + composeEntry.JobFinished = float64(compose.ImageBuilds[0].JobFinished.UnixNano()) / 1000000000 default: panic("invalid compose state") } @@ -64,7 +65,7 @@ func composeToComposeEntry(id uuid.UUID, compose store.Compose, includeUploads b return &composeEntry } -func composesToComposeEntries(composes map[uuid.UUID]store.Compose, uuids []uuid.UUID, includeUploads bool) []*ComposeEntry { +func composesToComposeEntries(composes map[uuid.UUID]compose.Compose, uuids []uuid.UUID, includeUploads bool) []*ComposeEntry { var composeEntries []*ComposeEntry if uuids == nil { composeEntries = make([]*ComposeEntry, 0, len(composes)) diff --git a/internal/weldr/upload.go b/internal/weldr/upload.go index 44ef52457..6c3c31b7b 100644 --- a/internal/weldr/upload.go +++ b/internal/weldr/upload.go @@ -3,6 +3,7 @@ package weldr import ( "encoding/json" "errors" + "github.com/osbuild/osbuild-composer/internal/common" "time" "github.com/google/uuid" @@ -10,12 +11,12 @@ import ( ) type UploadResponse struct { - Uuid uuid.UUID `json:"uuid"` - Status string `json:"status"` - ProviderName string `json:"provider_name"` - ImageName string `json:"image_name"` - CreationTime float64 `json:"creation_time"` - Settings target.TargetOptions `json:"settings"` + Uuid uuid.UUID `json:"uuid"` + Status common.ImageBuildState `json:"status"` + ProviderName string `json:"provider_name"` + ImageName string `json:"image_name"` + CreationTime float64 `json:"creation_time"` + Settings target.TargetOptions `json:"settings"` } type UploadRequest struct { @@ -103,7 +104,7 @@ func UploadRequestToTarget(u UploadRequest) (*target.Target, error) { t.ImageName = u.ImageName t.Options = u.Settings t.Name = targetName - t.Status = "WAITING" + t.Status = common.IBWaiting t.Created = time.Now() return &t, nil