particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

jobqueue: introduce the concept of channels

Browse source 
Channels are a concept similar to job types. Callers must specify a channel
name when queueing a new job. A list of channels is also specified when
dequeueing a job. The dequeued job's channel will always be from one of the
specified channel. Of course, the job types are also respected. The dequeued
job will also always be from one of the specified type.

Currently, all calls to jobqueue were changed so all queue operations use
an empty channel name and all dequeue operations use a list containing
an empty channel.

Thus, this is a non-functional change.

Signed-off-by: Ondřej Budai <ondrej@budai.cz>
This commit is contained in:
Ondřej Budai 2022-03-06 11:59:56 +01:00 • committed by Ondřej Budai
parent ba4f49078a
commit 7bfcee36f8
15 changed files with 202 additions and 176 deletions
Download patch file Download diff file Expand all files Collapse all files

24
internal/cloudapi/v2/v2.go
View file

@ -465,12 +465,12 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
var id uuid.UUID
if request.Koji != nil {
id, err = enqueueKojiCompose(h.server.workers, uint64(request.Koji.TaskId), request.Koji.Server, request.Koji.Name, request.Koji.Version, request.Koji.Release, distribution, bp, manifestSeed, irs)
id, err = enqueueKojiCompose(h.server.workers, uint64(request.Koji.TaskId), request.Koji.Server, request.Koji.Name, request.Koji.Version, request.Koji.Release, distribution, bp, manifestSeed, irs, "")
if err != nil {
return err
}
} else {
id, err = enqueueCompose(h.server.workers, distribution, bp, manifestSeed, irs)
id, err = enqueueCompose(h.server.workers, distribution, bp, manifestSeed, irs, "")
if err != nil {
return err
}
@ -488,7 +488,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
})
}
func enqueueCompose(workers *worker.Server, distribution distro.Distro, bp blueprint.Blueprint, manifestSeed int64, irs []imageRequest) (uuid.UUID, error) {
func enqueueCompose(workers *worker.Server, distribution distro.Distro, bp blueprint.Blueprint, manifestSeed int64, irs []imageRequest, channel string) (uuid.UUID, error) {
var id uuid.UUID
if len(irs) != 1 {
return id, HTTPError(ErrorInvalidNumberOfImageBuilds)
@ -502,12 +502,12 @@ func enqueueCompose(workers *worker.Server, distribution distro.Distro, bp bluep
Arch: ir.arch.Name(),
Releasever: distribution.Releasever(),
PackageSetsRepos: ir.packageSetsRepositories,
})
}, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}
manifestJobID, err := workers.EnqueueManifestJobByID(&worker.ManifestJobByID{}, depsolveJobID)
manifestJobID, err := workers.EnqueueManifestJobByID(&worker.ManifestJobByID{}, depsolveJobID, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}
@ -519,7 +519,7 @@ func enqueueCompose(workers *worker.Server, distribution distro.Distro, bp bluep
Build: ir.imageType.BuildPipelines(),
Payload: ir.imageType.PayloadPipelines(),
},
}, manifestJobID)
}, manifestJobID, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}
@ -530,7 +530,7 @@ func enqueueCompose(workers *worker.Server, distribution distro.Distro, bp bluep
return id, nil
}
func enqueueKojiCompose(workers *worker.Server, taskID uint64, server, name, version, release string, distribution distro.Distro, bp blueprint.Blueprint, manifestSeed int64, irs []imageRequest) (uuid.UUID, error) {
func enqueueKojiCompose(workers *worker.Server, taskID uint64, server, name, version, release string, distribution distro.Distro, bp blueprint.Blueprint, manifestSeed int64, irs []imageRequest, channel string) (uuid.UUID, error) {
var id uuid.UUID
kojiDirectory := "osbuild-composer-koji-" + uuid.New().String()
@ -539,7 +539,7 @@ func enqueueKojiCompose(workers *worker.Server, taskID uint64, server, name, ver
Name: name,
Version: version,
Release: release,
})
}, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}
@ -554,12 +554,12 @@ func enqueueKojiCompose(workers *worker.Server, taskID uint64, server, name, ver
Arch: ir.arch.Name(),
Releasever: distribution.Releasever(),
PackageSetsRepos: ir.packageSetsRepositories,
})
}, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}
manifestJobID, err := workers.EnqueueManifestJobByID(&worker.ManifestJobByID{}, depsolveJobID)
manifestJobID, err := workers.EnqueueManifestJobByID(&worker.ManifestJobByID{}, depsolveJobID, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}
@ -581,7 +581,7 @@ func enqueueKojiCompose(workers *worker.Server, taskID uint64, server, name, ver
KojiServer: server,
KojiDirectory: kojiDirectory,
KojiFilename: kojiFilename,
}, manifestJobID, initID)
}, manifestJobID, initID, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}
@ -600,7 +600,7 @@ func enqueueKojiCompose(workers *worker.Server, taskID uint64, server, name, ver
KojiDirectory: kojiDirectory,
TaskID: taskID,
StartTime: uint64(time.Now().Unix()),
}, initID, buildIDs)
}, initID, buildIDs, channel)
if err != nil {
return id, HTTPErrorWithInternal(ErrorEnqueueingJob, err)
}

17
internal/cloudapi/v2/v2_koji_test.go
View file

@ -12,6 +12,8 @@ import (
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
v2 "github.com/osbuild/osbuild-composer/internal/cloudapi/v2"
"github.com/osbuild/osbuild-composer/internal/distro/test_distro"
"github.com/osbuild/osbuild-composer/internal/kojiapi/api"
@ -19,7 +21,6 @@ import (
"github.com/osbuild/osbuild-composer/internal/test"
"github.com/osbuild/osbuild-composer/internal/worker"
"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
"github.com/stretchr/testify/require"
)
type jobResult struct {
@ -368,7 +369,7 @@ func TestKojiCompose(t *testing.T) {
c.composeReplyCode, c.composeReply, "id", "operation_id")
// handle koji-init
_, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"koji-init"})
_, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"koji-init"}, []string{""})
require.NoError(t, err)
require.Equal(t, "koji-init", jobType)
@ -386,7 +387,7 @@ func TestKojiCompose(t *testing.T) {
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
// handle osbuild-koji #1
_, token, jobType, rawJob, _, err = workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild-koji"})
_, token, jobType, rawJob, _, err = workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild-koji"}, []string{""})
require.NoError(t, err)
require.Equal(t, "osbuild-koji", jobType)
@ -403,7 +404,7 @@ func TestKojiCompose(t *testing.T) {
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
// handle osbuild-koji #2
_, token, jobType, rawJob, _, err = workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild-koji"})
_, token, jobType, rawJob, _, err = workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild-koji"}, []string{""})
require.NoError(t, err)
require.Equal(t, "osbuild-koji", jobType)
@ -427,7 +428,7 @@ func TestKojiCompose(t *testing.T) {
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
// handle koji-finalize
finalizeID, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"koji-finalize"})
finalizeID, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"koji-finalize"}, []string{""})
require.NoError(t, err)
require.Equal(t, "koji-finalize", jobType)
@ -510,7 +511,7 @@ func TestKojiJobTypeValidation(t *testing.T) {
Version: "42",
Release: "1",
}
initID, err := workers.EnqueueKojiInit(&initJob)
initID, err := workers.EnqueueKojiInit(&initJob, "")
require.NoError(t, err)
buildJobs := make([]worker.OSBuildKojiJob, nImages)
@ -524,7 +525,7 @@ func TestKojiJobTypeValidation(t *testing.T) {
KojiDirectory: "koji-server-test-dir",
KojiFilename: fname,
}
buildID, err := workers.EnqueueOSBuildKoji(fmt.Sprintf("fake-arch-%d", idx), &buildJob, initID)
buildID, err := workers.EnqueueOSBuildKoji(fmt.Sprintf("fake-arch-%d", idx), &buildJob, initID, "")
require.NoError(t, err)
buildJobs[idx] = buildJob
@ -542,7 +543,7 @@ func TestKojiJobTypeValidation(t *testing.T) {
TaskID: 0,
StartTime: uint64(time.Now().Unix()),
}
finalizeID, err := workers.EnqueueKojiFinalize(&finalizeJob, initID, buildJobIDs)
finalizeID, err := workers.EnqueueKojiFinalize(&finalizeJob, initID, buildJobIDs, "")
require.NoError(t, err)
// ----- Jobs queued - Test API endpoints (status, manifests, logs) ----- //

10
internal/cloudapi/v2/v2_test.go
View file

@ -39,7 +39,7 @@ func newV2Server(t *testing.T, dir string) (*v2.Server, *worker.Server, context.
depsolveContext, cancel := context.WithCancel(context.Background())
go func() {
for {
_, token, _, _, _, err := workerServer.RequestJob(context.Background(), test_distro.TestDistroName, []string{"depsolve"})
_, token, _, _, _, err := workerServer.RequestJob(context.Background(), test_distro.TestDistroName, []string{"depsolve"}, []string{""})
if err != nil {
continue
}
@ -575,7 +575,7 @@ func TestComposeStatusSuccess(t *testing.T) {
"kind": "ComposeId"
}`, "id")
jobId, token, jobType, args, dynArgs, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"})
jobId, token, jobType, args, dynArgs, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"}, []string{""})
require.NoError(t, err)
require.Equal(t, "osbuild", jobType)
@ -647,7 +647,7 @@ func TestComposeStatusFailure(t *testing.T) {
"kind": "ComposeId"
}`, "id")
jobId, token, jobType, _, _, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"})
jobId, token, jobType, _, _, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"}, []string{""})
require.NoError(t, err)
require.Equal(t, "osbuild", jobType)
@ -699,7 +699,7 @@ func TestComposeLegacyError(t *testing.T) {
"kind": "ComposeId"
}`, "id")
jobId, token, jobType, _, _, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"})
jobId, token, jobType, _, _, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"}, []string{""})
require.NoError(t, err)
require.Equal(t, "osbuild", jobType)
@ -754,7 +754,7 @@ func TestComposeJobError(t *testing.T) {
"kind": "ComposeId"
}`, "id")
jobId, token, jobType, _, _, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"})
jobId, token, jobType, _, _, err := wrksrv.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild"}, []string{""})
require.NoError(t, err)
require.Equal(t, "osbuild", jobType)