particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Weldr API: use the new compose/store API

Browse source 
The compose now contains multiple image builds, but Weldr API does not
support this feature. Use the first image build every time.

Also start using the new types instead of plain strings.
This commit is contained in:
Martin Sehnoutka 2020-02-10 09:43:49 +01:00 • committed by Ondřej Budai
parent 962365251d
commit ab6b6da76c
5 changed files with 193 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

91
internal/mocks/rpmmd/fixtures.go
View file

@ -2,12 +2,15 @@ package rpmmd_mock
import (
"fmt"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/compose"
"github.com/osbuild/osbuild-composer/internal/distro"
"sort"
"time"
"github.com/google/uuid"
"github.com/osbuild/osbuild-composer/internal/blueprint"
test_distro "github.com/osbuild/osbuild-composer/internal/distro/test"
test_distro "github.com/osbuild/osbuild-composer/internal/distro/fedoratest"
"github.com/osbuild/osbuild-composer/internal/rpmmd"
"github.com/osbuild/osbuild-composer/internal/store"
"github.com/osbuild/osbuild-composer/internal/target"
@ -71,7 +74,7 @@ func createBaseStoreFixture() *store.Store {
Name: "org.osbuild.local",
ImageName: "localimage",
Created: date,
Status: "WAITING",
Status: common.IBWaiting,
Options: &target.LocalTargetOptions{
Location: "/tmp/localimage",
},
@ -82,7 +85,7 @@ func createBaseStoreFixture() *store.Store {
Name: "org.osbuild.aws",
ImageName: "awsimage",
Created: date,
Status: "WAITING",
Status: common.IBWaiting,
Options: &target.AWSTargetOptions{
Region: "frankfurt",
AccessKeyID: "accesskey",
@ -93,42 +96,59 @@ func createBaseStoreFixture() *store.Store {
}
d := test_distro.New()
s := store.New(nil, d)
r := distro.NewRegistry([]string{"."})
s := store.New(nil, d, *r)
s.Blueprints[bName] = b
s.Composes = map[uuid.UUID]store.Compose{
uuid.MustParse("30000000-0000-0000-0000-000000000000"): store.Compose{
QueueStatus: "WAITING",
Blueprint: &b,
OutputType: "test_output",
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
s.Composes = map[uuid.UUID]compose.Compose{
uuid.MustParse("30000000-0000-0000-0000-000000000000"): compose.Compose{
Blueprint: &b,
ImageBuilds: []compose.ImageBuild{
{
QueueStatus: common.IBWaiting,
ImageType: common.Qcow2Generic,
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
},
},
},
uuid.MustParse("30000000-0000-0000-0000-000000000001"): store.Compose{
QueueStatus: "RUNNING",
Blueprint: &b,
OutputType: "test_output",
Targets: []*target.Target{localTarget},
JobCreated: date,
JobStarted: date,
uuid.MustParse("30000000-0000-0000-0000-000000000001"): compose.Compose{
Blueprint: &b,
ImageBuilds: []compose.ImageBuild{
{
QueueStatus: common.IBRunning,
ImageType: common.Qcow2Generic,
Targets: []*target.Target{localTarget},
JobCreated: date,
JobStarted: date,
},
},
},
uuid.MustParse("30000000-0000-0000-0000-000000000002"): store.Compose{
QueueStatus: "FINISHED",
Blueprint: &b,
OutputType: "test_output",
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
JobStarted: date,
JobFinished: date,
uuid.MustParse("30000000-0000-0000-0000-000000000002"): compose.Compose{
Blueprint: &b,
ImageBuilds: []compose.ImageBuild{
{
QueueStatus: common.IBFinished,
ImageType: common.Qcow2Generic,
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
JobStarted: date,
JobFinished: date,
},
},
},
uuid.MustParse("30000000-0000-0000-0000-000000000003"): store.Compose{
QueueStatus: "FAILED",
Blueprint: &b,
OutputType: "test_output",
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
JobStarted: date,
JobFinished: date,
uuid.MustParse("30000000-0000-0000-0000-000000000003"): compose.Compose{
Blueprint: &b,
ImageBuilds: []compose.ImageBuild{
{
QueueStatus: common.IBFailed,
ImageType: common.Qcow2Generic,
Targets: []*target.Target{localTarget, awsTarget},
JobCreated: date,
JobStarted: date,
JobFinished: date,
},
},
},
}
@ -166,7 +186,8 @@ func createStoreWithoutComposesFixture() *store.Store {
}
d := test_distro.New()
s := store.New(nil, d)
r := distro.NewRegistry([]string{"."})
s := store.New(nil, d, *r)
s.Blueprints[bName] = b

61
internal/weldr/api.go
View file

@ -4,7 +4,7 @@ import (
"archive/tar"
"bytes"
"encoding/json"
"errors"
errors_package "errors"
"fmt"
"io"
"log"
@ -356,7 +356,7 @@ func (api *API) sourceNewHandler(writer http.ResponseWriter, request *http.Reque
} else if contentType[0] == "text/x-toml" {
_, err = toml.DecodeReader(request.Body, &source)
} else {
err = errors.New("blueprint must be in json or toml format")
err = errors_package.New("blueprint must be in json or toml format")
}
if err != nil {
@ -1103,7 +1103,7 @@ func (api *API) blueprintsNewHandler(writer http.ResponseWriter, request *http.R
} else if contentType[0] == "text/x-toml" {
_, err = toml.DecodeReader(request.Body, &blueprint)
} else {
err = errors.New("blueprint must be in json or toml format")
err = errors_package.New("blueprint must be in json or toml format")
}
if err != nil {
@ -1143,7 +1143,7 @@ func (api *API) blueprintsWorkspaceHandler(writer http.ResponseWriter, request *
} else if contentType[0] == "text/x-toml" {
_, err = toml.DecodeReader(request.Body, &blueprint)
} else {
err = errors.New("blueprint must be in json or toml format")
err = errors_package.New("blueprint must be in json or toml format")
}
if err != nil {
@ -1211,6 +1211,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
BuildID uuid.UUID `json:"build_id"`
Status bool `json:"status"`
}
const MegaByte = 1024 * 1024
contentType := request.Header["Content-Type"]
if len(contentType) != 1 || contentType[0] != "application/json" {
@ -1256,6 +1257,13 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
bp := api.store.GetBlueprintCommitted(cr.BlueprintName)
size := cr.Size
// Microsoft Azure requires vhd images to be rounded up to the nearest MB
if cr.ComposeType == "vhd" && size%MegaByte != 0 {
size = (size/MegaByte + 1) * MegaByte
}
if bp != nil {
_, checksums, err := api.depsolveBlueprint(bp, true)
if err != nil {
@ -1267,7 +1275,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
return
}
err = api.store.PushCompose(reply.BuildID, bp, checksums, api.arch, cr.ComposeType, cr.Size, uploadTarget)
err = api.store.PushCompose(reply.BuildID, bp, checksums, api.arch, cr.ComposeType, size, uploadTarget)
// TODO: we should probably do some kind of blueprint validation in future
// for now, let's just 500 and bail out
@ -1386,10 +1394,10 @@ func (api *API) composeQueueHandler(writer http.ResponseWriter, request *http.Re
composes := api.store.GetAllComposes()
for id, compose := range composes {
switch compose.QueueStatus {
case "WAITING":
switch compose.GetState() {
case common.CWaiting:
reply.New = append(reply.New, composeToComposeEntry(id, compose, isRequestVersionAtLeast(params, 1)))
case "RUNNING":
case common.CRunning:
reply.Run = append(reply.Run, composeToComposeEntry(id, compose, isRequestVersionAtLeast(params, 1)))
}
}
@ -1428,7 +1436,6 @@ func (api *API) composeStatusHandler(writer http.ResponseWriter, request *http.R
}
}
composes := api.store.GetAllComposes()
reply.UUIDs = composesToComposeEntries(composes, uuids, isRequestVersionAtLeast(params, 1))
json.NewEncoder(writer).Encode(reply)
@ -1482,14 +1489,16 @@ func (api *API) composeInfoHandler(writer http.ResponseWriter, request *http.Req
reply.Deps = Dependencies{
Packages: make([]map[string]interface{}, 0),
}
reply.ComposeType = compose.OutputType
reply.QueueStatus = compose.QueueStatus
if compose.Image != nil {
reply.ImageSize = compose.Size
// Weldr API assumes only one image build per compose, that's why only the
// 1st build is considered
reply.ComposeType, _ = compose.ImageBuilds[0].ImageType.ToCompatString()
reply.QueueStatus = compose.GetState().ToString()
if compose.ImageBuilds[0].Image != nil {
reply.ImageSize = compose.ImageBuilds[0].Size
}
if isRequestVersionAtLeast(params, 1) {
reply.Uploads = TargetsToUploadResponses(compose.Targets)
reply.Uploads = TargetsToUploadResponses(compose.ImageBuilds[0].Targets)
}
json.NewEncoder(writer).Encode(reply)
@ -1521,16 +1530,16 @@ func (api *API) composeImageHandler(writer http.ResponseWriter, request *http.Re
return
}
if compose.QueueStatus != "FINISHED" {
if compose.GetState() != common.CFinished {
errors := responseError{
ID: "BuildInWrongState",
Msg: fmt.Sprintf("Build %s is in wrong state: %s", uuidString, compose.QueueStatus),
Msg: fmt.Sprintf("Build %s is in wrong state: %s", uuidString, compose.GetState().ToString()),
}
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
if compose.Image == nil {
if compose.ImageBuilds[0].Image == nil {
errors := responseError{
ID: "BuildMissingFile",
Msg: fmt.Sprintf("Compose %s doesn't have an image assigned", uuidString),
@ -1539,9 +1548,9 @@ func (api *API) composeImageHandler(writer http.ResponseWriter, request *http.Re
return
}
imageName := filepath.Base(compose.Image.Path)
imageName := filepath.Base(compose.ImageBuilds[0].Image.Path)
file, err := os.Open(compose.Image.Path)
file, err := os.Open(compose.ImageBuilds[0].Image.Path)
if err != nil {
errors := responseError{
ID: "BuildMissingFile",
@ -1552,8 +1561,8 @@ func (api *API) composeImageHandler(writer http.ResponseWriter, request *http.Re
}
writer.Header().Set("Content-Disposition", "attachment; filename="+uuid.String()+"-"+imageName)
writer.Header().Set("Content-Type", compose.Image.Mime)
writer.Header().Set("Content-Length", fmt.Sprintf("%d", compose.Image.Size))
writer.Header().Set("Content-Type", compose.ImageBuilds[0].Image.Mime)
writer.Header().Set("Content-Length", fmt.Sprintf("%d", compose.ImageBuilds[0].Image.Size))
io.Copy(writer, file)
}
@ -1584,7 +1593,7 @@ func (api *API) composeLogsHandler(writer http.ResponseWriter, request *http.Req
return
}
if compose.QueueStatus != "FINISHED" && compose.QueueStatus != "FAILED" {
if compose.GetState() != common.CFinished && compose.GetState() != common.CFailed {
errors := responseError{
ID: "BuildInWrongState",
Msg: fmt.Sprintf("Build %s not in FINISHED or FAILED state.", uuidString),
@ -1664,7 +1673,7 @@ func (api *API) composeLogHandler(writer http.ResponseWriter, request *http.Requ
return
}
if compose.QueueStatus == "WAITING" {
if compose.GetState() == common.CWaiting {
errors := responseError{
ID: "BuildInWrongState",
Msg: fmt.Sprintf("Build %s has not started yet. No logs to view.", uuidString),
@ -1673,7 +1682,7 @@ func (api *API) composeLogHandler(writer http.ResponseWriter, request *http.Requ
return
}
if compose.QueueStatus == "RUNNING" {
if compose.GetState() == common.CRunning {
fmt.Fprintf(writer, "Running...\n")
return
}
@ -1717,7 +1726,7 @@ func (api *API) composeFinishedHandler(writer http.ResponseWriter, request *http
composes := api.store.GetAllComposes()
for _, entry := range composesToComposeEntries(composes, nil, isRequestVersionAtLeast(params, 1)) {
switch entry.QueueStatus {
case "FINISHED":
case common.IBFinished:
reply.Finished = append(reply.Finished, entry)
}
}
@ -1737,7 +1746,7 @@ func (api *API) composeFailedHandler(writer http.ResponseWriter, request *http.R
composes := api.store.GetAllComposes()
for _, entry := range composesToComposeEntries(composes, nil, isRequestVersionAtLeast(params, 1)) {
switch entry.QueueStatus {
case "FAILED":
case common.IBFailed:
reply.Failed = append(reply.Failed, entry)
}
}

113
internal/weldr/api_test.go
View file

@ -3,6 +3,9 @@ package weldr_test
import (
"archive/tar"
"bytes"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/compose"
"github.com/osbuild/osbuild-composer/internal/target"
"io"
"math/rand"
"net/http"
@ -13,11 +16,9 @@ import (
"time"
"github.com/osbuild/osbuild-composer/internal/blueprint"
_ "github.com/osbuild/osbuild-composer/internal/distro/test"
test_distro "github.com/osbuild/osbuild-composer/internal/distro/test"
test_distro "github.com/osbuild/osbuild-composer/internal/distro/fedoratest"
rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd"
"github.com/osbuild/osbuild-composer/internal/store"
"github.com/osbuild/osbuild-composer/internal/target"
"github.com/osbuild/osbuild-composer/internal/test"
"github.com/osbuild/osbuild-composer/internal/weldr"
@ -31,7 +32,7 @@ func createWeldrAPI(fixtureGenerator rpmmd_mock.FixtureGenerator) (*weldr.API, *
rpm := rpmmd_mock.NewRPMMDMock(fixture)
d := test_distro.New()
return weldr.New(rpm, "test_arch", d, nil, fixture.Store), fixture.Store
return weldr.New(rpm, "x86_64", d, nil, fixture.Store), fixture.Store
}
func TestBasic(t *testing.T) {
@ -47,8 +48,8 @@ func TestBasic(t *testing.T) {
{"/api/v0/projects/source/info", http.StatusNotFound, ``},
{"/api/v0/projects/source/info/", http.StatusNotFound, `{"errors":[{"code":404,"id":"HTTPError","msg":"Not Found"}],"status":false}`},
{"/api/v0/projects/source/info/foo", http.StatusOK, `{"errors":[{"id":"UnknownSource","msg":"foo is not a valid source"}],"sources":{}}`},
{"/api/v0/projects/source/info/test-id", http.StatusOK, `{"sources":{"test-id":{"name":"test-id","type":"yum-baseurl","url":"http://example.com/test/os/test_arch","check_gpg":true,"check_ssl":true,"system":true}},"errors":[]}`},
{"/api/v0/projects/source/info/*", http.StatusOK, `{"sources":{"test-id":{"name":"test-id","type":"yum-baseurl","url":"http://example.com/test/os/test_arch","check_gpg":true,"check_ssl":true,"system":true}},"errors":[]}`},
{"/api/v0/projects/source/info/test-id", http.StatusOK, `{"sources":{"test-id":{"name":"test-id","type":"yum-baseurl","url":"http://example.com/test/os/x86_64","check_gpg":true,"check_ssl":true,"system":true}},"errors":[]}`},
{"/api/v0/projects/source/info/*", http.StatusOK, `{"sources":{"test-id":{"name":"test-id","type":"yum-baseurl","url":"http://example.com/test/os/x86_64","check_gpg":true,"check_ssl":true,"system":true}},"errors":[]}`},
{"/api/v0/blueprints/list", http.StatusOK, `{"total":1,"offset":0,"limit":1,"blueprints":["test"]}`},
{"/api/v0/blueprints/info/", http.StatusNotFound, `{"errors":[{"code":404,"id":"HTTPError","msg":"Not Found"}],"status":false}`},
@ -293,8 +294,7 @@ func TestBlueprintsDepsolve(t *testing.T) {
}
func TestCompose(t *testing.T) {
expectedComposeLocal := &store.Compose{
QueueStatus: "WAITING",
expectedComposeLocal := &compose.Compose{
Blueprint: &blueprint.Blueprint{
Name: "test",
Version: "0.0.0",
@ -303,31 +303,40 @@ func TestCompose(t *testing.T) {
Groups: []blueprint.Group{},
Customizations: nil,
},
OutputType: "test_output",
Targets: []*target.Target{},
}
expectedComposeLocalAndAws := &store.Compose{
QueueStatus: "WAITING",
Blueprint: &blueprint.Blueprint{
Name: "test",
Version: "0.0.0",
Packages: []blueprint.Package{},
Modules: []blueprint.Package{},
Groups: []blueprint.Group{},
Customizations: nil,
},
OutputType: "test_output",
Targets: []*target.Target{
ImageBuilds: []compose.ImageBuild{
{
Name: "org.osbuild.aws",
Status: "WAITING",
ImageName: "test_upload",
Options: &target.AWSTargetOptions{
Region: "frankfurt",
AccessKeyID: "accesskey",
SecretAccessKey: "secretkey",
Bucket: "clay",
Key: "imagekey",
QueueStatus: common.IBWaiting,
ImageType: common.Qcow2Generic,
Targets: []*target.Target{},
},
},
}
expectedComposeLocalAndAws := &compose.Compose{
Blueprint: &blueprint.Blueprint{
Name: "test",
Version: "0.0.0",
Packages: []blueprint.Package{},
Modules: []blueprint.Package{},
Groups: []blueprint.Group{},
Customizations: nil,
},
ImageBuilds: []compose.ImageBuild{
{
QueueStatus: common.IBWaiting,
ImageType: common.Qcow2Generic,
Targets: []*target.Target{
{
Name: "org.osbuild.aws",
Status: common.IBWaiting,
ImageName: "test_upload",
Options: &target.AWSTargetOptions{
Region: "frankfurt",
AccessKeyID: "accesskey",
SecretAccessKey: "secretkey",
Bucket: "clay",
Key: "imagekey",
},
},
},
},
},
@ -340,12 +349,12 @@ func TestCompose(t *testing.T) {
Body string
ExpectedStatus int
ExpectedJSON string
ExpectedCompose *store.Compose
ExpectedCompose *compose.Compose
IgnoreFields []string
}{
{true, "POST", "/api/v0/compose", `{"blueprint_name": "http-server","compose_type": "test_output","branch": "master"}`, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownBlueprint","msg":"Unknown blueprint name: http-server"}]}`, nil, []string{"build_id"}},
{false, "POST", "/api/v0/compose", `{"blueprint_name": "test","compose_type": "test_output","branch": "master"}`, http.StatusOK, `{"status": true}`, expectedComposeLocal, []string{"build_id"}},
{false, "POST", "/api/v1/compose", `{"blueprint_name": "test","compose_type":"test_output","branch":"master","upload":{"image_name":"test_upload","provider":"aws","settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}}`, http.StatusOK, `{"status": true}`, expectedComposeLocalAndAws, []string{"build_id"}},
{true, "POST", "/api/v0/compose", `{"blueprint_name": "http-server","compose_type": "qcow2","branch": "master"}`, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownBlueprint","msg":"Unknown blueprint name: http-server"}]}`, nil, []string{"build_id"}},
{false, "POST", "/api/v0/compose", `{"blueprint_name": "test","compose_type": "qcow2","branch": "master"}`, http.StatusOK, `{"status": true}`, expectedComposeLocal, []string{"build_id"}},
{false, "POST", "/api/v1/compose", `{"blueprint_name": "test","compose_type":"qcow2","branch":"master","upload":{"image_name":"test_upload","provider":"aws","settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}}`, http.StatusOK, `{"status": true}`, expectedComposeLocalAndAws, []string{"build_id"}},
}
for _, c := range cases {
@ -361,20 +370,20 @@ func TestCompose(t *testing.T) {
}
// I have no idea how to get the compose in better way
var compose store.Compose
var composeStruct compose.Compose
for _, c := range s.Composes {
compose = c
composeStruct = c
break
}
if compose.Pipeline == nil {
if composeStruct.ImageBuilds[0].Pipeline == nil {
t.Fatalf("%s: the compose in the store did not contain a blueprint", c.Path)
} else {
// TODO: find some (reasonable) way to verify the contents of the pipeline
compose.Pipeline = nil
composeStruct.ImageBuilds[0].Pipeline = nil
}
if diff := cmp.Diff(compose, *c.ExpectedCompose, test.IgnoreDates(), test.IgnoreUuids(), test.Ignore("Targets.Options.Location")); diff != "" {
if diff := cmp.Diff(composeStruct, *c.ExpectedCompose, test.IgnoreDates(), test.IgnoreUuids(), test.Ignore("Targets.Options.Location")); diff != "" {
t.Errorf("%s: compose in store isn't the same as expected, diff:\n%s", c.Path, diff)
}
@ -425,9 +434,9 @@ func TestComposeStatus(t *testing.T) {
ExpectedStatus int
ExpectedJSON string
}{
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/30000000-0000-0000-0000-000000000000,30000000-0000-0000-0000-000000000002", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/*", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000001","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"RUNNING","job_created":1574857140,"job_started":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140},{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/status/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING","job_created":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/30000000-0000-0000-0000-000000000000,30000000-0000-0000-0000-000000000002", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/status/*", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING","job_created":1574857140},{"id":"30000000-0000-0000-0000-000000000001","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"RUNNING","job_created":1574857140,"job_started":1574857140},{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140},{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/status/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"uuids":[{"id":"30000000-0000-0000-0000-000000000000","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING","job_created":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`},
}
if len(os.Getenv("OSBUILD_COMPOSER_TEST_EXTERNAL")) > 0 {
@ -449,8 +458,8 @@ func TestComposeInfo(t *testing.T) {
ExpectedStatus int
ExpectedJSON string
}{
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/info/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"id":"30000000-0000-0000-0000-000000000000","config":"","blueprint":{"name":"test","description":"","version":"0.0.0","packages":[],"modules":[],"groups":[]},"commit":"","deps":{"packages":[]},"compose_type":"test_output","queue_status":"WAITING","image_size":0}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/info/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"id":"30000000-0000-0000-0000-000000000000","config":"","blueprint":{"name":"test","description":"","version":"0.0.0","packages":[],"modules":[],"groups":[]},"commit":"","deps":{"packages":[]},"compose_type":"test_output","queue_status":"WAITING","image_size":0,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/info/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"id":"30000000-0000-0000-0000-000000000000","config":"","blueprint":{"name":"test","description":"","version":"0.0.0","packages":[],"modules":[],"groups":[]},"commit":"","deps":{"packages":[]},"compose_type":"qcow2","queue_status":"WAITING","image_size":0}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/info/30000000-0000-0000-0000-000000000000", ``, http.StatusOK, `{"id":"30000000-0000-0000-0000-000000000000","config":"","blueprint":{"name":"test","description":"","version":"0.0.0","packages":[],"modules":[],"groups":[]},"commit":"","deps":{"packages":[]},"compose_type":"qcow2","queue_status":"WAITING","image_size":0,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/info/30000000-0000-0000-0000", ``, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownUUID","msg":"30000000-0000-0000-0000 is not a valid build uuid"}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/info/42000000-0000-0000-0000-000000000000", ``, http.StatusBadRequest, `{"status":false,"errors":[{"id":"UnknownUUID","msg":"42000000-0000-0000-0000-000000000000 is not a valid build uuid"}]}`},
}
@ -571,8 +580,8 @@ func TestComposeQueue(t *testing.T) {
ExpectedStatus int
ExpectedJSON string
}{
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING"}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"RUNNING"}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"WAITING","uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"RUNNING"}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING"}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"RUNNING"}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/queue", ``, http.StatusOK, `{"new":[{"blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"WAITING","uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}],"run":[{"blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"RUNNING"}]}`},
{rpmmd_mock.NoComposesFixture, "GET", "/api/v0/compose/queue", ``, http.StatusOK, `{"new":[],"run":[]}`},
}
@ -595,8 +604,8 @@ func TestComposeFinished(t *testing.T) {
ExpectedStatus int
ExpectedJSON string
}{
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/finished", ``, http.StatusOK, `{"finished":[{"id":"30000000-0000-0000-0000-000000000002","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FINISHED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`},
{rpmmd_mock.NoComposesFixture, "GET", "/api/v0/compose/finished", ``, http.StatusOK, `{"finished":[]}`},
}
@ -619,8 +628,8 @@ func TestComposeFailed(t *testing.T) {
ExpectedStatus int
ExpectedJSON string
}{
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"test_output","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v0/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140}]}`},
{rpmmd_mock.BaseFixture, "GET", "/api/v1/compose/failed", ``, http.StatusOK, `{"failed":[{"id":"30000000-0000-0000-0000-000000000003","blueprint":"test","version":"0.0.0","compose_type":"qcow2","image_size":0,"queue_status":"FAILED","job_created":1574857140,"job_started":1574857140,"job_finished":1574857140,"uploads":[{"uuid":"10000000-0000-0000-0000-000000000000","status":"WAITING","provider_name":"aws","image_name":"awsimage","creation_time":1574857140,"settings":{"region":"frankfurt","accessKeyID":"accesskey","secretAccessKey":"secretkey","bucket":"clay","key":"imagekey"}}]}]}`},
{rpmmd_mock.NoComposesFixture, "GET", "/api/v0/compose/failed", ``, http.StatusOK, `{"failed":[]}`},
}

65
internal/weldr/compose.go
View file

@ -2,61 +2,62 @@ package weldr
import (
"github.com/google/uuid"
"github.com/osbuild/osbuild-composer/internal/store"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/compose"
"log"
"sort"
)
type ComposeEntry struct {
ID uuid.UUID `json:"id"`
Blueprint string `json:"blueprint"`
Version string `json:"version"`
ComposeType string `json:"compose_type"`
ImageSize uint64 `json:"image_size"`
QueueStatus string `json:"queue_status"`
JobCreated float64 `json:"job_created"`
JobStarted float64 `json:"job_started,omitempty"`
JobFinished float64 `json:"job_finished,omitempty"`
Uploads []UploadResponse `json:"uploads,omitempty"`
ID uuid.UUID `json:"id"`
Blueprint string `json:"blueprint"`
Version string `json:"version"`
ComposeType common.ImageType `json:"compose_type"`
ImageSize uint64 `json:"image_size"`
QueueStatus common.ImageBuildState `json:"queue_status"`
JobCreated float64 `json:"job_created"`
JobStarted float64 `json:"job_started,omitempty"`
JobFinished float64 `json:"job_finished,omitempty"`
Uploads []UploadResponse `json:"uploads,omitempty"`
}
func composeToComposeEntry(id uuid.UUID, compose store.Compose, includeUploads bool) *ComposeEntry {
func composeToComposeEntry(id uuid.UUID, compose compose.Compose, includeUploads bool) *ComposeEntry {
var composeEntry ComposeEntry
composeEntry.ID = id
composeEntry.Blueprint = compose.Blueprint.Name
composeEntry.Version = compose.Blueprint.Version
composeEntry.ComposeType = compose.OutputType
composeEntry.QueueStatus = compose.QueueStatus
composeEntry.ComposeType = compose.ImageBuilds[0].ImageType
composeEntry.QueueStatus = compose.ImageBuilds[0].QueueStatus
if includeUploads {
composeEntry.Uploads = TargetsToUploadResponses(compose.Targets)
composeEntry.Uploads = TargetsToUploadResponses(compose.ImageBuilds[0].Targets)
}
switch compose.QueueStatus {
case "WAITING":
composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000
switch compose.ImageBuilds[0].QueueStatus {
case common.IBWaiting:
composeEntry.JobCreated = float64(compose.ImageBuilds[0].JobCreated.UnixNano()) / 1000000000
case "RUNNING":
composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000
composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000
case common.IBRunning:
composeEntry.JobCreated = float64(compose.ImageBuilds[0].JobCreated.UnixNano()) / 1000000000
composeEntry.JobStarted = float64(compose.ImageBuilds[0].JobStarted.UnixNano()) / 1000000000
case "FINISHED":
if compose.Image != nil {
composeEntry.ImageSize = compose.Size
case common.IBFinished:
if compose.ImageBuilds[0].Image != nil {
composeEntry.ImageSize = compose.ImageBuilds[0].Size
} else {
log.Printf("finished compose with id %s has nil image\n", id.String())
composeEntry.ImageSize = 0
}
composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000
composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000
composeEntry.JobFinished = float64(compose.JobFinished.UnixNano()) / 1000000000
composeEntry.JobCreated = float64(compose.ImageBuilds[0].JobCreated.UnixNano()) / 1000000000
composeEntry.JobStarted = float64(compose.ImageBuilds[0].JobStarted.UnixNano()) / 1000000000
composeEntry.JobFinished = float64(compose.ImageBuilds[0].JobFinished.UnixNano()) / 1000000000
case "FAILED":
composeEntry.JobCreated = float64(compose.JobCreated.UnixNano()) / 1000000000
composeEntry.JobStarted = float64(compose.JobStarted.UnixNano()) / 1000000000
composeEntry.JobFinished = float64(compose.JobFinished.UnixNano()) / 1000000000
case common.IBFailed:
composeEntry.JobCreated = float64(compose.ImageBuilds[0].JobCreated.UnixNano()) / 1000000000
composeEntry.JobStarted = float64(compose.ImageBuilds[0].JobStarted.UnixNano()) / 1000000000
composeEntry.JobFinished = float64(compose.ImageBuilds[0].JobFinished.UnixNano()) / 1000000000
default:
panic("invalid compose state")
}
@ -64,7 +65,7 @@ func composeToComposeEntry(id uuid.UUID, compose store.Compose, includeUploads b
return &composeEntry
}
func composesToComposeEntries(composes map[uuid.UUID]store.Compose, uuids []uuid.UUID, includeUploads bool) []*ComposeEntry {
func composesToComposeEntries(composes map[uuid.UUID]compose.Compose, uuids []uuid.UUID, includeUploads bool) []*ComposeEntry {
var composeEntries []*ComposeEntry
if uuids == nil {
composeEntries = make([]*ComposeEntry, 0, len(composes))

15
internal/weldr/upload.go
View file

@ -3,6 +3,7 @@ package weldr
import (
"encoding/json"
"errors"
"github.com/osbuild/osbuild-composer/internal/common"
"time"
"github.com/google/uuid"
@ -10,12 +11,12 @@ import (
)
type UploadResponse struct {
Uuid uuid.UUID `json:"uuid"`
Status string `json:"status"`
ProviderName string `json:"provider_name"`
ImageName string `json:"image_name"`
CreationTime float64 `json:"creation_time"`
Settings target.TargetOptions `json:"settings"`
Uuid uuid.UUID `json:"uuid"`
Status common.ImageBuildState `json:"status"`
ProviderName string `json:"provider_name"`
ImageName string `json:"image_name"`
CreationTime float64 `json:"creation_time"`
Settings target.TargetOptions `json:"settings"`
}
type UploadRequest struct {
@ -103,7 +104,7 @@ func UploadRequestToTarget(u UploadRequest) (*target.Target, error) {
t.ImageName = u.ImageName
t.Options = u.Settings
t.Name = targetName
t.Status = "WAITING"
t.Status = common.IBWaiting
t.Created = time.Now()
return &t, nil