particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

jobqueue: include image build id

Browse source 
The compose will soon move to a concept of including multiple image
builds per one compose, we need to accommodate extra identifier to
handle this scenario.
This commit is contained in:
Martin Sehnoutka 2020-02-10 09:36:09 +01:00 • committed by Ondřej Budai
parent 73994eb06d
commit 80e01bb397
3 changed files with 44 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

18
internal/jobqueue/api.go
View file

@ -89,15 +89,17 @@ func (api *API) addJobHandler(writer http.ResponseWriter, request *http.Request,
return return
} }
nextJob := api.store.PopCompose() nextJob := api.store.PopJob()
writer.WriteHeader(http.StatusCreated) writer.WriteHeader(http.StatusCreated)
json.NewEncoder(writer).Encode(Job{ // FIXME: handle or comment this possible error
ID: nextJob.ComposeID, _ = json.NewEncoder(writer).Encode(Job{
Distro: nextJob.Distro, ID: nextJob.ComposeID,
Pipeline: nextJob.Pipeline, ImageBuildID: nextJob.ImageBuildID,
Targets: nextJob.Targets, Distro: nextJob.Distro,
OutputType: nextJob.OutputType, Pipeline: nextJob.Pipeline,
Targets: nextJob.Targets,
OutputType: nextJob.ImageType,
}) })
} }
@ -121,7 +123,7 @@ func (api *API) updateJobHandler(writer http.ResponseWriter, request *http.Reque
return return
} }
err = api.store.UpdateCompose(id, body.Status, body.Image, body.Result) err = api.store.UpdateImageBuildInCompose(id, body.ImageBuildID, body.Status, body.Image, body.Result)
if err != nil { if err != nil {
switch err.(type) { switch err.(type) {
case *store.NotFoundError: case *store.NotFoundError:

26
internal/jobqueue/api_test.go
View file

@ -5,7 +5,8 @@ import (
"testing" "testing"
"github.com/osbuild/osbuild-composer/internal/blueprint" "github.com/osbuild/osbuild-composer/internal/blueprint"
test_distro "github.com/osbuild/osbuild-composer/internal/distro/test" "github.com/osbuild/osbuild-composer/internal/distro"
test_distro "github.com/osbuild/osbuild-composer/internal/distro/fedoratest"
"github.com/osbuild/osbuild-composer/internal/jobqueue" "github.com/osbuild/osbuild-composer/internal/jobqueue"
"github.com/osbuild/osbuild-composer/internal/store" "github.com/osbuild/osbuild-composer/internal/store"
"github.com/osbuild/osbuild-composer/internal/test" "github.com/osbuild/osbuild-composer/internal/test"
@ -28,12 +29,13 @@ func TestBasic(t *testing.T) {
// Update job that does not exist, with invalid body // Update job that does not exist, with invalid body
{"PATCH", "/job-queue/v1/jobs/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", ``, http.StatusBadRequest, ``}, {"PATCH", "/job-queue/v1/jobs/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", ``, http.StatusBadRequest, ``},
// Update job that does not exist // Update job that does not exist
{"PATCH", "/job-queue/v1/jobs/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", `{"status":"RUNNING"}`, http.StatusNotFound, ``}, {"PATCH", "/job-queue/v1/jobs/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", `{"image_build_id": 0, "status":"RUNNING"}`, http.StatusNotFound, ``},
} }
for _, c := range cases { for _, c := range cases {
distro := test_distro.New() distroStruct := test_distro.New()
api := jobqueue.New(nil, store.New(nil, distro)) registry := distro.NewRegistry([]string{"."})
api := jobqueue.New(nil, store.New(nil, distroStruct, *registry))
test.TestRoute(t, api, false, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON) test.TestRoute(t, api, false, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON)
} }
@ -41,27 +43,29 @@ func TestBasic(t *testing.T) {
func TestCreate(t *testing.T) { func TestCreate(t *testing.T) {
id, _ := uuid.Parse("ffffffff-ffff-ffff-ffff-ffffffffffff") id, _ := uuid.Parse("ffffffff-ffff-ffff-ffff-ffffffffffff")
distro := test_distro.New() distroStruct := test_distro.New()
store := store.New(nil, distro) registry := distro.NewRegistry([]string{"."})
store := store.New(nil, distroStruct, *registry)
api := jobqueue.New(nil, store) api := jobqueue.New(nil, store)
err := store.PushCompose(id, &blueprint.Blueprint{}, map[string]string{"test-repo": "test:foo"}, "test_arch", "test_output", 0, nil) err := store.PushCompose(id, &blueprint.Blueprint{}, map[string]string{"test-repo": "test:foo"}, "x86_64", "qcow2", 0, nil)
if err != nil { if err != nil {
t.Fatalf("error pushing compose: %v", err) t.Fatalf("error pushing compose: %v", err)
} }
test.TestRoute(t, api, false, "POST", "/job-queue/v1/jobs", `{}`, http.StatusCreated, test.TestRoute(t, api, false, "POST", "/job-queue/v1/jobs", `{}`, http.StatusCreated,
`{"distro":"test-distro","id":"ffffffff-ffff-ffff-ffff-ffffffffffff","output_type":"test_output","pipeline":{},"targets":[]}`, "created", "uuid") `{"distro":"fedora-30","id":"ffffffff-ffff-ffff-ffff-ffffffffffff","image_build_id":0,"output_type":"qcow2","pipeline":{},"targets":[]}`, "created", "uuid")
} }
func testUpdateTransition(t *testing.T, from, to string, expectedStatus int) { func testUpdateTransition(t *testing.T, from, to string, expectedStatus int) {
id, _ := uuid.Parse("ffffffff-ffff-ffff-ffff-ffffffffffff") id, _ := uuid.Parse("ffffffff-ffff-ffff-ffff-ffffffffffff")
distro := test_distro.New() distroStruct := test_distro.New()
store := store.New(nil, distro) registry := distro.NewRegistry([]string{"."})
store := store.New(nil, distroStruct, *registry)
api := jobqueue.New(nil, store) api := jobqueue.New(nil, store)
if from != "VOID" { if from != "VOID" {
err := store.PushCompose(id, &blueprint.Blueprint{}, map[string]string{"test": "test:foo"}, "test_arch", "test_output", 0, nil) err := store.PushCompose(id, &blueprint.Blueprint{}, map[string]string{"test": "test:foo"}, "x86_64", "qcow2", 0, nil)
if err != nil { if err != nil {
t.Fatalf("error pushing compose: %v", err) t.Fatalf("error pushing compose: %v", err)
} }

33
internal/jobqueue/job.go
View file

@ -3,6 +3,7 @@ package jobqueue
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/osbuild/osbuild-composer/internal/compose"
"io/ioutil" "io/ioutil"
"os" "os"
"os/exec" "os/exec"
@ -12,23 +13,24 @@ import (
"github.com/osbuild/osbuild-composer/internal/common" "github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/distro" "github.com/osbuild/osbuild-composer/internal/distro"
"github.com/osbuild/osbuild-composer/internal/pipeline" "github.com/osbuild/osbuild-composer/internal/pipeline"
"github.com/osbuild/osbuild-composer/internal/store"
"github.com/osbuild/osbuild-composer/internal/target" "github.com/osbuild/osbuild-composer/internal/target"
"github.com/osbuild/osbuild-composer/internal/upload/awsupload" "github.com/osbuild/osbuild-composer/internal/upload/awsupload"
) )
type Job struct { type Job struct {
ID uuid.UUID `json:"id"` ID uuid.UUID `json:"id"`
Distro string `json:"distro"` ImageBuildID int `json:"image_build_id"`
Pipeline *pipeline.Pipeline `json:"pipeline"` Distro string `json:"distro"`
Targets []*target.Target `json:"targets"` Pipeline *pipeline.Pipeline `json:"pipeline"`
OutputType string `json:"output_type"` Targets []*target.Target `json:"targets"`
OutputType string `json:"output_type"`
} }
type JobStatus struct { type JobStatus struct {
Status string `json:"status"` Status common.ImageBuildState `json:"status"`
Image *store.Image `json:"image"` ImageBuildID int `json:"image_build_id"`
Result *common.ComposeResult `json:"result"` Image *compose.Image `json:"image"`
Result *common.ComposeResult `json:"result"`
} }
type TargetsError struct { type TargetsError struct {
@ -45,7 +47,7 @@ func (e *TargetsError) Error() string {
return errString return errString
} }
func (job *Job) Run() (*store.Image, *common.ComposeResult, error) { func (job *Job) Run() (*compose.Image, *common.ComposeResult, error) {
distros := distro.NewRegistry([]string{"/etc/osbuild-composer", "/usr/share/osbuild-composer"}) distros := distro.NewRegistry([]string{"/etc/osbuild-composer", "/usr/share/osbuild-composer"})
d := distros.GetDistro(job.Distro) d := distros.GetDistro(job.Distro)
if d == nil { if d == nil {
@ -60,6 +62,7 @@ func (job *Job) Run() (*store.Image, *common.ComposeResult, error) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
// FIXME: how to handle errors in defer?
defer os.Remove(buildFile.Name()) defer os.Remove(buildFile.Name())
err = json.NewEncoder(buildFile).Encode(build) err = json.NewEncoder(buildFile).Encode(build)
@ -71,6 +74,7 @@ func (job *Job) Run() (*store.Image, *common.ComposeResult, error) {
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("error setting up osbuild store: %v", err) return nil, nil, fmt.Errorf("error setting up osbuild store: %v", err)
} }
// FIXME: how to handle errors in defer?
defer os.RemoveAll(tmpStore) defer os.RemoveAll(tmpStore)
cmd := exec.Command( cmd := exec.Command(
@ -100,7 +104,8 @@ func (job *Job) Run() (*store.Image, *common.ComposeResult, error) {
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("error encoding osbuild pipeline: %v", err) return nil, nil, fmt.Errorf("error encoding osbuild pipeline: %v", err)
} }
stdin.Close() // FIXME: handle or comment this possible error
_ = stdin.Close()
var result common.ComposeResult var result common.ComposeResult
err = json.NewDecoder(stdout).Decode(&result) err = json.NewDecoder(stdout).Decode(&result)
@ -118,7 +123,7 @@ func (job *Job) Run() (*store.Image, *common.ComposeResult, error) {
return nil, &result, fmt.Errorf("cannot fetch information about output type %s: %v", job.OutputType, err) return nil, &result, fmt.Errorf("cannot fetch information about output type %s: %v", job.OutputType, err)
} }
var image store.Image var image compose.Image
var r []error var r []error
@ -150,10 +155,10 @@ func (job *Job) Run() (*store.Image, *common.ComposeResult, error) {
return nil, &result, err return nil, &result, err
} }
image = store.Image{ image = compose.Image{
Path: imagePath, Path: imagePath,
Mime: mimeType, Mime: mimeType,
Size: fileStat.Size(), Size: uint64(fileStat.Size()),
} }
case *target.AWSTargetOptions: case *target.AWSTargetOptions: