diff --git a/internal/client/compose.go b/internal/client/compose.go new file mode 100644 index 000000000..b2ff9eaa1 --- /dev/null +++ b/internal/client/compose.go @@ -0,0 +1,188 @@ +// Package client - compose contains functions for the compose API +// Copyright (C) 2020 by Red Hat, Inc. +package client + +import ( + "encoding/json" + "io" + "net/http" + "strings" + + "github.com/osbuild/osbuild-composer/internal/weldr" +) + +// PostComposeV0 sends a JSON compose string to the API +// and returns an APIResponse +func PostComposeV0(socket *http.Client, compose string) (*APIResponse, error) { + body, resp, err := PostJSON(socket, "/api/v0/compose", compose) + if resp != nil || err != nil { + return resp, err + } + return NewAPIResponse(body) +} + +// NewComposeResponseV0 converts the response body to a status response +func NewComposeResponseV0(body []byte) (*weldr.ComposeResponseV0, error) { + var response weldr.ComposeResponseV0 + err := json.Unmarshal(body, &response) + if err != nil { + return nil, err + } + return &response, nil +} + +// GetFinishedComposesV0 returns a list of the finished composes +func GetFinishedComposesV0(socket *http.Client) ([]weldr.ComposeEntryV0, *APIResponse, error) { + body, resp, err := GetRaw(socket, "GET", "/api/v0/compose/finished") + if resp != nil || err != nil { + return []weldr.ComposeEntryV0{}, resp, err + } + var finished weldr.ComposeFinishedResponseV0 + err = json.Unmarshal(body, &finished) + if err != nil { + return []weldr.ComposeEntryV0{}, nil, err + } + return finished.Finished, nil, nil +} + +// GetFailedComposesV0 returns a list of the failed composes +func GetFailedComposesV0(socket *http.Client) ([]weldr.ComposeEntryV0, *APIResponse, error) { + body, resp, err := GetRaw(socket, "GET", "/api/v0/compose/failed") + if resp != nil || err != nil { + return []weldr.ComposeEntryV0{}, resp, err + } + var failed weldr.ComposeFailedResponseV0 + err = json.Unmarshal(body, &failed) + if err != nil { + return []weldr.ComposeEntryV0{}, nil, err + } + return failed.Failed, nil, nil +} + +// GetComposeStatusV0 returns a list of composes matching the optional filter parameters +func GetComposeStatusV0(socket *http.Client, uuids, blueprint, status, composeType string) ([]weldr.ComposeEntryV0, *APIResponse, error) { + // Build the query string + route := "/api/v0/compose/status/" + uuids + var filters []string + + if len(blueprint) > 0 { + filters = append(filters, "blueprint="+blueprint) + } + if len(status) > 0 { + filters = append(filters, "status="+status) + } + if len(composeType) > 0 { + filters = append(filters, "type="+composeType) + } + + if len(filters) > 0 { + route = route + "?" + strings.Join(filters, "&") + } + + body, resp, err := GetRaw(socket, "GET", route) + if resp != nil || err != nil { + return []weldr.ComposeEntryV0{}, resp, err + } + var composes weldr.ComposeStatusResponseV0 + err = json.Unmarshal(body, &composes) + if err != nil { + return []weldr.ComposeEntryV0{}, nil, err + } + return composes.UUIDs, nil, nil +} + +// GetComposeTypesV0 returns a list of the failed composes +func GetComposesTypesV0(socket *http.Client) ([]weldr.ComposeTypeV0, *APIResponse, error) { + body, resp, err := GetRaw(socket, "GET", "/api/v0/compose/types") + if resp != nil || err != nil { + return []weldr.ComposeTypeV0{}, resp, err + } + var composeTypes weldr.ComposeTypesResponseV0 + err = json.Unmarshal(body, &composeTypes) + if err != nil { + return []weldr.ComposeTypeV0{}, nil, err + } + return composeTypes.Types, nil, nil +} + +// DeleteComposeV0 deletes one or more composes based on their uuid +func DeleteComposeV0(socket *http.Client, uuids string) (weldr.DeleteComposeResponseV0, *APIResponse, error) { + body, resp, err := DeleteRaw(socket, "/api/v0/compose/delete/"+uuids) + if resp != nil || err != nil { + return weldr.DeleteComposeResponseV0{}, resp, err + } + var deleteResponse weldr.DeleteComposeResponseV0 + err = json.Unmarshal(body, &deleteResponse) + if err != nil { + return weldr.DeleteComposeResponseV0{}, nil, err + } + return deleteResponse, nil, nil +} + +// GetComposeInfoV0 returns detailed information about the selected compose +func GetComposeInfoV0(socket *http.Client, uuid string) (weldr.ComposeInfoResponseV0, *APIResponse, error) { + body, resp, err := GetRaw(socket, "GET", "/api/v0/compose/info/"+uuid) + if resp != nil || err != nil { + return weldr.ComposeInfoResponseV0{}, resp, err + } + var info weldr.ComposeInfoResponseV0 + err = json.Unmarshal(body, &info) + if err != nil { + return weldr.ComposeInfoResponseV0{}, nil, err + } + return info, nil, nil +} + +// GetComposeQueueV0 returns the list of composes in the queue +func GetComposeQueueV0(socket *http.Client) (weldr.ComposeQueueResponseV0, *APIResponse, error) { + body, resp, err := GetRaw(socket, "GET", "/api/v0/compose/queue") + if resp != nil || err != nil { + return weldr.ComposeQueueResponseV0{}, resp, err + } + var queue weldr.ComposeQueueResponseV0 + err = json.Unmarshal(body, &queue) + if err != nil { + return weldr.ComposeQueueResponseV0{}, nil, err + } + return queue, nil, nil +} + +// Test compose metadata for unknown uuid + +// Test compose results for unknown uuid + +// WriteComposeImageV0 requests the image for a compose and writes it to an io.Writer +func WriteComposeImageV0(socket *http.Client, w io.Writer, uuid string) (*APIResponse, error) { + body, resp, err := GetRawBody(socket, "GET", "/api/v0/compose/image/"+uuid) + if resp != nil || err != nil { + return resp, err + } + _, err = io.Copy(w, body) + body.Close() + + return nil, err +} + +// WriteComposeLogsV0 requests the logs for a compose and writes it to an io.Writer +func WriteComposeLogsV0(socket *http.Client, w io.Writer, uuid string) (*APIResponse, error) { + body, resp, err := GetRawBody(socket, "GET", "/api/v0/compose/logs/"+uuid) + if resp != nil || err != nil { + return resp, err + } + _, err = io.Copy(w, body) + body.Close() + + return nil, err +} + +// WriteComposeLogV0 requests the log for a compose and writes it to an io.Writer +func WriteComposeLogV0(socket *http.Client, w io.Writer, uuid string) (*APIResponse, error) { + body, resp, err := GetRawBody(socket, "GET", "/api/v0/compose/log/"+uuid) + if resp != nil || err != nil { + return resp, err + } + _, err = io.Copy(w, body) + body.Close() + + return nil, err +} diff --git a/internal/client/compose_test.go b/internal/client/compose_test.go new file mode 100644 index 000000000..de73cd040 --- /dev/null +++ b/internal/client/compose_test.go @@ -0,0 +1,352 @@ +// Package client - compose_test contains functions to check the compose API +// Copyright (C) 2020 by Red Hat, Inc. + +// Tests should be self-contained and not depend on the state of the server +// They should use their own blueprints, not the default blueprints +// They should not assume version numbers for packages will match +// They should run tests that depend on previous results from the same function +// not from other functions. +// +// NOTE: The compose fail/finish tests use fake composes so the following are not +// fully tested here: +// +// * image download +// * log download +// * logs archive download +// +// In addition osbuild-composer has not implemented: +// +// * compose/results +// * compose/metadata +package client + +import ( + "io/ioutil" + "net/http" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/osbuild/osbuild-composer/internal/weldr" +) + +// Test the compose types API +func TestComposeTypesV0(t *testing.T) { + composeTypes, resp, err := GetComposesTypesV0(testState.socket) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.Greater(t, len(composeTypes), 0) + var found bool + for _, t := range composeTypes { + if t.Name == "qcow2" && t.Enabled == true { + found = true + break + } + } + require.True(t, found, "qcow2 not in list of compose types: %#v", composeTypes) +} + +// Test compose with invalid type fails +func TestComposeInvalidTypeV0(t *testing.T) { + // lorax-composer checks the blueprint name before checking the compose type + // so we need to push an empty blueprint to make sure the right failure is checked + bp := ` + name="test-compose-invalid-type-v0" + description="TestComposeInvalidTypeV0" + version="0.0.1" + ` + resp, err := PostTOMLBlueprintV0(testState.socket, bp) + require.NoError(t, err, "failed with a client error") + require.True(t, resp.Status, "POST failed: %#v", resp) + + compose := `{ + "blueprint_name": "test-compose-invalid-type-v0", + "compose_type": "snakes", + "branch": "master" + }` + resp, err = PostComposeV0(testState.socket, compose) + require.NoError(t, err, "failed with a client error") + require.NotNil(t, resp) + require.False(t, resp.Status, "POST did not fail") + require.Equal(t, len(resp.Errors), 1) + require.Contains(t, resp.Errors[0].Msg, "snakes") +} + +// Test compose for unknown blueprint fails +func TestComposeInvalidBlueprintV0(t *testing.T) { + compose := `{ + "blueprint_name": "test-invalid-bp-compose-v0", + "compose_type": "qcow2", + "branch": "master" + }` + resp, err := PostComposeV0(testState.socket, compose) + require.NoError(t, err, "failed with a client error") + require.NotNil(t, resp) + require.False(t, resp.Status, "POST did not fail") + require.Equal(t, len(resp.Errors), 1) + require.Contains(t, resp.Errors[0].Msg, "test-invalid-bp-compose-v0") +} + +// Test compose cancel for unknown uuid fails +// Is cancel implemented at all? + +// Test compose delete for unknown uuid +func TestDeleteUnknownComposeV0(t *testing.T) { + status, resp, err := DeleteComposeV0(testState.socket, "c91818f9-8025-47af-89d2-f030d7000c2c") + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + // TODO -- fix the API Handler in osbuild-composer, should be no uuids + assert.Equal(t, 0, len(status.UUIDs), "%#v", status) + require.Equal(t, 1, len(status.Errors), "%#v", status) + require.Equal(t, "UnknownUUID", status.Errors[0].ID) + require.Contains(t, status.Errors[0].Msg, "c91818f9-8025-47af-89d2-f030d7000c2c") +} + +// Test compose info for unknown uuid +func TestUnknownComposeInfoV0(t *testing.T) { + _, resp, err := GetComposeInfoV0(testState.socket, "c91818f9-8025-47af-89d2-f030d7000c2c") + require.NoError(t, err, "failed with a client error") + require.NotNil(t, resp) + require.False(t, resp.Status) + require.Equal(t, 1, len(resp.Errors)) + require.Equal(t, "UnknownUUID", resp.Errors[0].ID) + require.Contains(t, resp.Errors[0].Msg, "c91818f9-8025-47af-89d2-f030d7000c2c") +} + +// Test compose metadata for unknown uuid +// TODO osbuild-composer has not implemented compose/metadata yet + +// Test compose results for unknown uuid +// TODO osbuild-composer has not implemented compose/results yet + +// Test compose image for unknown uuid +func TestComposeInvalidImageV0(t *testing.T) { + resp, err := WriteComposeImageV0(testState.socket, ioutil.Discard, "c91818f9-8025-47af-89d2-f030d7000c2c") + require.NoError(t, err, "failed with a client error") + require.NotNil(t, resp) + require.False(t, resp.Status) + require.Equal(t, 1, len(resp.Errors)) + require.Equal(t, "UnknownUUID", resp.Errors[0].ID) + require.Contains(t, resp.Errors[0].Msg, "c91818f9-8025-47af-89d2-f030d7000c2c") +} + +// Test compose logs for unknown uuid +func TestComposeInvalidLogsV0(t *testing.T) { + resp, err := WriteComposeLogsV0(testState.socket, ioutil.Discard, "c91818f9-8025-47af-89d2-f030d7000c2c") + require.NoError(t, err, "failed with a client error") + require.NotNil(t, resp) + require.False(t, resp.Status) + require.Equal(t, 1, len(resp.Errors)) + require.Equal(t, "UnknownUUID", resp.Errors[0].ID) + require.Contains(t, resp.Errors[0].Msg, "c91818f9-8025-47af-89d2-f030d7000c2c") +} + +// Test compose log for unknown uuid +func TestComposeInvalidLogV0(t *testing.T) { + resp, err := WriteComposeLogV0(testState.socket, ioutil.Discard, "c91818f9-8025-47af-89d2-f030d7000c2c") + require.NoError(t, err, "failed with a client error") + require.NotNil(t, resp) + require.False(t, resp.Status) + require.Equal(t, 1, len(resp.Errors)) + require.Equal(t, "UnknownUUID", resp.Errors[0].ID) + require.Contains(t, resp.Errors[0].Msg, "c91818f9-8025-47af-89d2-f030d7000c2c") +} + +// Test status filter for unknown uuid +func TestComposeInvalidStatusV0(t *testing.T) { + status, resp, err := GetComposeStatusV0(testState.socket, "c91818f9-8025-47af-89d2-f030d7000c2c", "", "", "") + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.Equal(t, 0, len(status)) +} + +// Helper for searching compose results for a UUID +func UUIDInComposeResults(buildID uuid.UUID, results []weldr.ComposeEntryV0) bool { + for idx := range results { + if results[idx].ID == buildID { + return true + } + } + return false +} + +// Helper to wait for a build id to not be in the queue +func WaitForBuild(socket *http.Client, buildID uuid.UUID) (*APIResponse, error) { + for { + queue, resp, err := GetComposeQueueV0(testState.socket) + if err != nil { + return nil, err + } + if resp != nil { + return resp, nil + } + + if !UUIDInComposeResults(buildID, queue.New) && + !UUIDInComposeResults(buildID, queue.Run) { + break + } + } + return nil, nil +} + +// Setup and run the failed compose tests +func TestFailedComposeV0(t *testing.T) { + bp := ` + name="test-failed-compose-v0" + description="TestFailedComposeV0" + version="0.0.1" + [[packages]] + name="bash" + version="*" + + [[modules]] + name="util-linux" + version="*" + + [[customizations.user]] + name="root" + password="qweqweqwe" + ` + resp, err := PostTOMLBlueprintV0(testState.socket, bp) + require.NoError(t, err, "failed with a client error") + require.True(t, resp.Status, "POST failed: %#v", resp) + + compose := `{ + "blueprint_name": "test-failed-compose-v0", + "compose_type": "qcow2", + "branch": "master" + }` + // Create a failed test compose + body, resp, err := PostJSON(testState.socket, "/api/v1/compose?test=1", compose) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + + response, err := NewComposeResponseV0(body) + require.NoError(t, err, "failed with a client error") + require.True(t, response.Status, "POST failed: %#v", response) + buildID := response.BuildID + + // Wait until the build is not listed in the queue + resp, err = WaitForBuild(testState.socket, buildID) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + + // Test finished after compose (should not have finished) + finished, resp, err := GetFinishedComposesV0(testState.socket) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.False(t, UUIDInComposeResults(buildID, finished)) + + // Test failed after compose (should have failed) + failed, resp, err := GetFailedComposesV0(testState.socket) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.True(t, UUIDInComposeResults(buildID, failed), "%s not found in failed list: %#v", buildID, failed) + + // Test status filter on failed compose + status, resp, err := GetComposeStatusV0(testState.socket, "*", "", "FAILED", "") + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.True(t, UUIDInComposeResults(buildID, status), "%s not found in status list: %#v", buildID, status) + + // Test status of build id + status, resp, err = GetComposeStatusV0(testState.socket, buildID.String(), "", "", "") + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.True(t, UUIDInComposeResults(buildID, status), "%s not found in status list: %#v", buildID, status) + + // Test status filter using FINISHED, should not be listed + status, resp, err = GetComposeStatusV0(testState.socket, "*", "", "FINISHED", "") + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.False(t, UUIDInComposeResults(buildID, status)) + + // Test compose info for the failed compose + info, resp, err := GetComposeInfoV0(testState.socket, buildID.String()) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.Equal(t, "FAILED", info.QueueStatus) + require.Equal(t, buildID, info.ID) +} + +// Setup and run the finished compose tests +func TestFinishedComposeV0(t *testing.T) { + bp := ` + name="test-finished-compose-v0" + description="TestFinishedComposeV0" + version="0.0.1" + [[packages]] + name="bash" + version="*" + + [[modules]] + name="util-linux" + version="*" + + [[customizations.user]] + name="root" + password="qweqweqwe" + ` + resp, err := PostTOMLBlueprintV0(testState.socket, bp) + require.NoError(t, err, "failed with a client error") + require.True(t, resp.Status, "POST failed: %#v", resp) + + compose := `{ + "blueprint_name": "test-finished-compose-v0", + "compose_type": "qcow2", + "branch": "master" + }` + // Create a finished test compose + body, resp, err := PostJSON(testState.socket, "/api/v1/compose?test=2", compose) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + + response, err := NewComposeResponseV0(body) + require.NoError(t, err, "failed with a client error") + require.True(t, response.Status, "POST failed: %#v", response) + buildID := response.BuildID + + // Wait until the build is not listed in the queue + resp, err = WaitForBuild(testState.socket, buildID) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + + // Test failed after compose (should not have failed) + failed, resp, err := GetFailedComposesV0(testState.socket) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.False(t, UUIDInComposeResults(buildID, failed)) + + // Test finished after compose (should have finished) + finished, resp, err := GetFinishedComposesV0(testState.socket) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.True(t, UUIDInComposeResults(buildID, finished), "%s not found in finished list: %#v", buildID, finished) + + // Test status filter on finished compose + status, resp, err := GetComposeStatusV0(testState.socket, "*", "", "FINISHED", "") + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.True(t, UUIDInComposeResults(buildID, status), "%s not found in status list: %#v", buildID, status) + + // Test status of build id + status, resp, err = GetComposeStatusV0(testState.socket, buildID.String(), "", "", "") + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.True(t, UUIDInComposeResults(buildID, status), "%s not found in status list: %#v", buildID, status) + + // Test status filter using FAILED, should not be listed + status, resp, err = GetComposeStatusV0(testState.socket, "*", "", "FAILED", "") + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.False(t, UUIDInComposeResults(buildID, status)) + + // Test compose info for the finished compose + info, resp, err := GetComposeInfoV0(testState.socket, buildID.String()) + require.NoError(t, err, "failed with a client error") + require.Nil(t, resp) + require.Equal(t, "FINISHED", info.QueueStatus) + require.Equal(t, buildID, info.ID) +} diff --git a/internal/weldr/json.go b/internal/weldr/json.go index ed767e243..81d7511fa 100644 --- a/internal/weldr/json.go +++ b/internal/weldr/json.go @@ -3,7 +3,10 @@ package weldr import ( + "github.com/google/uuid" + "github.com/osbuild/osbuild-composer/internal/blueprint" + "github.com/osbuild/osbuild-composer/internal/common" "github.com/osbuild/osbuild-composer/internal/rpmmd" "github.com/osbuild/osbuild-composer/internal/store" ) @@ -156,3 +159,75 @@ type ModulesListV0 struct { type ModulesInfoV0 struct { Modules []rpmmd.PackageInfo `json:"modules"` } + +type ComposeRequestV0 struct { + BlueprintName string `json:"blueprint_name"` + ComposeType string `json:"compose_type"` + Branch string `json:"branch"` +} +type ComposeResponseV0 struct { + BuildID uuid.UUID `json:"build_id"` + Status bool `json:"status"` +} + +// This is similar to weldr.ComposeEntry but different because internally the image types are capitalized +type ComposeEntryV0 struct { + ID uuid.UUID `json:"id"` + Blueprint string `json:"blueprint"` + Version string `json:"version"` + ComposeType string `json:"compose_type"` + ImageSize uint64 `json:"image_size"` // This is user-provided image size, not actual file size + QueueStatus common.ImageBuildState `json:"queue_status"` + JobCreated float64 `json:"job_created"` + JobStarted float64 `json:"job_started,omitempty"` + JobFinished float64 `json:"job_finished,omitempty"` + Uploads []uploadResponse `json:"uploads,omitempty"` +} + +type ComposeFinishedResponseV0 struct { + Finished []ComposeEntryV0 `json:"finished"` +} +type ComposeFailedResponseV0 struct { + Failed []ComposeEntryV0 `json:"failed"` +} +type ComposeStatusResponseV0 struct { + UUIDs []ComposeEntryV0 `json:"uuids"` +} + +type ComposeTypeV0 struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` +} + +type ComposeTypesResponseV0 struct { + Types []ComposeTypeV0 `json:"types"` +} + +type DeleteComposeStatusV0 struct { + UUID uuid.UUID `json:"uuid"` + Status bool `json:"status"` +} + +type DeleteComposeResponseV0 struct { + UUIDs []DeleteComposeStatusV0 `json:"uuids"` + Errors []ResponseError `json:"errors"` +} + +// NOTE: This does not include the lorax-composer specific 'config' field +type ComposeInfoResponseV0 struct { + ID uuid.UUID `json:"id"` + Blueprint *blueprint.Blueprint `json:"blueprint"` // blueprint not frozen! + Commit string `json:"commit"` // empty for now + Deps struct { + Packages []rpmmd.Package `json:"packages"` + } `json:"deps"` + ComposeType string `json:"compose_type"` + QueueStatus string `json:"queue_status"` + ImageSize uint64 `json:"image_size"` + Uploads []uploadResponse `json:"uploads,omitempty"` +} + +type ComposeQueueResponseV0 struct { + New []ComposeEntryV0 `json:"new"` + Run []ComposeEntryV0 `json:"run"` +}