particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rcm: rework the API for submitting composes

Browse source 
The API allowed composes to have multiple architectures, image
types and repositories. Turns out that's not exactly what we want
it is not clear how to combine the lits of each. Each architecture
might not combine with each image type, and it is not clear which
repositories are need for each image build. Lastly, while allowing
different image builds to have different distros in the same
compose does not appear immediately useful, there is no particular
reason to disallow that. This patch reworks the way composes are
specified.

The intention remains the same, to be able to submit several image
builds as one compose. But rather than taking arrays of image types
and architectures, take one array of image builds instead, each of
which consists of one distro, one architecture, one image build and
an array of repositories.

In a follow-up patch they will also each contain an array of upload
targets.

This means that each image build will have the same sort of structure
as a compose request in the weldr API. The reason we want to submit
an array of them rather than have them as individual composes, is that
in a follow-up patch we will introduce the concept of a "finalizer",
or "call-back" or something to that effect, which will be triggered
when all the images have been built successfully. The use-case is,
as always, koji, which requires this.

Signed-off-by: Tom Gundersen <teg@jklm.no>
This commit is contained in:
Tom Gundersen 2020-03-27 22:38:40 +01:00
parent 1c0d951f54
commit fa015a70f6
3 changed files with 226 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

73
internal/rcm/api.go
View file

@ -8,7 +8,6 @@ import (
"log"
"net"
"net/http"
"strings"
"github.com/osbuild/osbuild-composer/internal/rpmmd"
@ -108,15 +107,22 @@ func (api *API) submit(writer http.ResponseWriter, request *http.Request, _ http
}
type repository struct {
URL string `json:"url"`
BaseURL string `json:"baseurl,omitempty"`
Metalink string `json:"metalink,omitempty"`
MirrorList string `json:"mirrorlist,omitempty"`
GPGKey string `json:"gpgkey,omitempty"`
}
type imageBuild struct {
Distribution string `json:"distribution"`
Architecture string `json:"architecture"`
ImageType string `json:"image_type"`
Repositories []repository `json:"repositories"`
}
// JSON structure expected from the client
var composeRequest struct {
Distribution string `json:"distribution"`
ImageTypes []string `json:"image_types"`
Architectures []string `json:"architectures"`
Repositories []repository `json:"repositories"`
ImageBuilds []imageBuild `json:"image_builds"`
}
// JSON structure with error message
var errorReason struct {
@ -126,35 +132,27 @@ func (api *API) submit(writer http.ResponseWriter, request *http.Request, _ http
decoder := json.NewDecoder(request.Body)
decoder.DisallowUnknownFields()
err := decoder.Decode(&composeRequest)
if err != nil || len(composeRequest.Architectures) != 1 || len(composeRequest.ImageTypes) != 1 || len(composeRequest.Repositories) == 0 {
if err != nil {
writer.WriteHeader(http.StatusBadRequest)
errors := []string{}
err = json.NewEncoder(writer).Encode(err.Error())
if err != nil {
errors = append(errors, err.Error())
}
if len(composeRequest.ImageTypes) == 0 {
errors = append(errors, "input must specify an image type")
} else if len(composeRequest.ImageTypes) != 1 {
errors = append(errors, "multiple image types are not yet supported")
}
if len(composeRequest.Architectures) == 0 {
errors = append(errors, "input must specify an architecture")
} else if len(composeRequest.Architectures) != 1 {
errors = append(errors, "multiple architectures are not yet supported")
}
if len(composeRequest.Repositories) == 0 {
errors = append(errors, "input must specify repositories")
}
errorReason.Error = strings.Join(errors, ", ")
err = json.NewEncoder(writer).Encode(errorReason)
if err != nil {
// JSON encoding is clearly our fault.
panic("Failed to encode errors in RCM API. This is a bug.")
panic("Failed to write response")
}
return
}
distro := api.distros.GetDistro(composeRequest.Distribution)
if len(composeRequest.ImageBuilds) != 1 {
writer.WriteHeader(http.StatusBadRequest)
_, err := writer.Write([]byte("unsupported number of image builds"))
if err != nil {
panic("Failed to write response")
}
return
}
buildRequest := composeRequest.ImageBuilds[0]
distro := api.distros.GetDistro(buildRequest.Distribution)
if distro == nil {
writer.WriteHeader(http.StatusBadRequest)
_, err := writer.Write([]byte("unknown distro"))
@ -164,7 +162,7 @@ func (api *API) submit(writer http.ResponseWriter, request *http.Request, _ http
return
}
arch, err := distro.GetArch(composeRequest.Architectures[0])
arch, err := distro.GetArch(buildRequest.Architecture)
if err != nil {
writer.WriteHeader(http.StatusBadRequest)
_, err := writer.Write([]byte("unknown architecture for distro"))
@ -174,7 +172,7 @@ func (api *API) submit(writer http.ResponseWriter, request *http.Request, _ http
return
}
imageType, err := arch.GetImageType(composeRequest.ImageTypes[0])
imageType, err := arch.GetImageType(buildRequest.ImageType)
if err != nil {
writer.WriteHeader(http.StatusBadRequest)
_, err := writer.Write([]byte("unknown image type for distro and architecture"))
@ -184,14 +182,15 @@ func (api *API) submit(writer http.ResponseWriter, request *http.Request, _ http
return
}
// Create repo configurations from the URLs in the request. Use made up repo id and name, because
// we don't want to bother clients of this API with details like this
// Create repo configurations from the URLs in the request.
repoConfigs := []rpmmd.RepoConfig{}
for n, repo := range composeRequest.Repositories {
for n, repo := range buildRequest.Repositories {
repoConfigs = append(repoConfigs, rpmmd.RepoConfig{
Id: fmt.Sprintf("repo-%d", n),
BaseURL: repo.URL,
IgnoreSSL: false,
Id: fmt.Sprintf("repo-%d", n),
BaseURL: repo.BaseURL,
Metalink: repo.Metalink,
MirrorList: repo.MirrorList,
GPGKey: repo.GPGKey,
})
}

204
internal/rcm/api_test.go
View file

@ -3,6 +3,7 @@ package rcm_test
import (
"bytes"
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
"regexp"
@ -44,7 +45,7 @@ func TestBasicRcmAPI(t *testing.T) {
{"GET", "/v1/compose", ``, "", http.StatusMethodNotAllowed, ``},
{"POST", "/v1/compose", `{"status":"RUNNING"}`, "application/json", http.StatusBadRequest, ``},
{"POST", "/v1/compose", `{"status":"RUNNING"}`, "text/plain", http.StatusBadRequest, ``},
{"POST", "/v1/compose", `{"distribution": "fedora-30", "image_types": ["qcow2"], "architectures":["x86_64"], "repositories": []}`, "application/json", http.StatusBadRequest, ""},
{"POST", "/v1/compose", `{"image_builds":[]}`, "application/json", http.StatusBadRequest, ""},
{"POST", "/v1/compose/111-222-333", `{"status":"RUNNING"}`, "application/json", http.StatusMethodNotAllowed, ``},
{"GET", "/v1/compose/7802c476-9cd1-41b7-ba81-43c1906bce73", `{"status":"RUNNING"}`, "application/json", http.StatusBadRequest, `{"error_reason":"Compose UUID does not exist"}`},
}
@ -73,7 +74,134 @@ func TestBasicRcmAPI(t *testing.T) {
}
}
func TestSubmitCompose(t *testing.T) {
func TestSubmit(t *testing.T) {
registry, err := distro_mock.NewDefaultRegistry()
if err != nil {
t.Fatal(err)
}
api := rcm.New(nil, store.New(nil), rpmmd_mock.NewRPMMDMock(rpmmd_mock.BaseFixture()), registry)
var submit_reply struct {
UUID uuid.UUID `json:"compose_id"`
}
var cases = []struct {
Method string
Path string
Body string
ContentType string
ExpectedStatus int
}{
{
"POST",
"/v1/compose",
`{
"image_builds":
[
{
"distribution": "fedora-30",
"architecture": "x86_64",
"image_type": "qcow2",
"repositories":
[
{
"baseurl": "http://download.fedoraproject.org/pub/fedora/linux/releases/30/Everything/x86_64/os/"
}
]
}
]
}`,
"application/json",
http.StatusOK,
},
{
"POST",
"/v1/compose",
`{
"image_builds":
[
{
"distribution": "invalid",
"architecture": "x86_64",
"image_type": "qcow2",
"repositories":
[
{
"baseurl": "http://download.fedoraproject.org/pub/fedora/linux/releases/30/Everything/x86_64/os/"
}
]
}
]
}`,
"application/json",
http.StatusBadRequest,
},
{
"POST",
"/v1/compose",
`{
"image_builds":
[
{
"distribution": "fedora-30",
"architecture": "invalid",
"image_type": "qcow2",
"repositories":
[
{
"baseurl": "http://download.fedoraproject.org/pub/fedora/linux/releases/30/Everything/x86_64/os/"
}
]
}
]
}`,
"application/json",
http.StatusBadRequest,
},
{
"POST",
"/v1/compose",
`{
"image_builds":
[
{
"distribution": "fedora-30",
"architecture": "x86_64",
"image_type": "invalid",
"repositories":
[
{
"baseurl": "http://download.fedoraproject.org/pub/fedora/linux/releases/30/Everything/x86_64/os/"
}
]
}
]
}`,
"application/json",
http.StatusBadRequest,
},
}
for n, c := range cases {
// Submit job
t.Logf("RCM API submit compose, case %d\n", n)
resp := internalRequest(api, c.Method, c.Path, c.Body, c.ContentType)
if resp.StatusCode != c.ExpectedStatus {
errReason, _ := ioutil.ReadAll(resp.Body)
t.Fatal("Failed to call /v1/compose, HTTP status code: ", resp.StatusCode, "Error message: ", string(errReason))
}
if resp.StatusCode == http.StatusOK {
decoder := json.NewDecoder(resp.Body)
decoder.DisallowUnknownFields()
err := decoder.Decode(&submit_reply)
if err != nil {
t.Fatal("Failed to decode response to /v1/compose:", err)
}
}
}
}
func TestStatus(t *testing.T) {
// Test the most basic use case: Submit a new job and get its status.
registry, err := distro_mock.NewDefaultRegistry()
if err != nil {
@ -88,50 +216,66 @@ func TestSubmitCompose(t *testing.T) {
Status string `json:"status,omitempty"`
ErrorReason string `json:"error_reason,omitempty"`
}
// Submit a job
resp := internalRequest(api,
"POST",
"/v1/compose",
`{
"image_builds":
[
{
"distribution": "fedora-30",
"architecture": "x86_64",
"image_type": "qcow2",
"repositories":
[
{
"baseurl": "http://download.fedoraproject.org/pub/fedora/linux/releases/30/Everything/x86_64/os/"
}
]
}
]
}`,
"application/json")
if resp.StatusCode != http.StatusOK {
errReason, _ := ioutil.ReadAll(resp.Body)
t.Fatal("Failed to call /v1/compose, HTTP status code: ", resp.StatusCode, "Error message: ", string(errReason))
}
decoder := json.NewDecoder(resp.Body)
decoder.DisallowUnknownFields()
err = decoder.Decode(&submit_reply)
if err != nil {
t.Fatal("Failed to decode response to /v1/compose:", err)
}
var cases = []struct {
Method string
Path string
Body string
ContentType string
Method string
Path string
Body string
ContentType string
ExpectedStatus string
}{
{
"POST",
"/v1/compose",
`{"distribution": "fedora-30",
"image_types": ["qcow2"],
"architectures":["x86_64"],
"repositories": [{
"url": "http://download.fedoraproject.org/pub/fedora/linux/releases/30/Everything/x86_64/os/"
}]}`,
"GET",
"/v1/compose/" + submit_reply.UUID.String(),
``,
"application/json",
"WAITING",
},
}
for n, c := range cases {
// Submit job
t.Logf("RCM API submit compose, case %d\n", n)
resp := internalRequest(api, c.Method, c.Path, c.Body, c.ContentType)
if resp.StatusCode != http.StatusOK {
t.Fatal("Failed to call /v1/compose, HTTP status code:", resp.StatusCode)
}
decoder := json.NewDecoder(resp.Body)
decoder.DisallowUnknownFields()
err := decoder.Decode(&submit_reply)
if err != nil {
t.Fatal("Failed to decode response to /v1/compose:", err)
}
// Get the status
t.Log("RCM API get status")
resp = internalRequest(api, "GET", "/v1/compose/"+submit_reply.UUID.String(), "", "")
t.Logf("RCM API get compose status, case %d\n", n)
resp = internalRequest(api, c.Method, c.Path, c.Body, c.ContentType)
decoder = json.NewDecoder(resp.Body)
decoder.DisallowUnknownFields()
err = decoder.Decode(&status_reply)
if err != nil {
t.Fatal("Failed to decode response to /v1/compose/UUID:", err)
}
if status_reply.ErrorReason != "" {
t.Error("Failed to get compose status, reason:", status_reply.ErrorReason)
if status_reply.Status != c.ExpectedStatus {
t.Error("Failed to get compose status:", status_reply.Status)
}
}
}