debian-forge-composer/internal/cloudapi/v2/v2_koji_test.go
Tomáš Hozza efbaa93eef Target/Koji: rework result options to contain more than just image info
The `KojiTargetResultOptions` previously contained information only
about the uploaded image file. And even then, some information, such as
the filename, were scattered in other structures such as
`KojiFinalizeJob` struct.

Since the plan is to start uploading also osbuild manifest and osbuild
build log to Koji, we need to extend the result options structure to
hold more information and also make it specific to which file is the
information related.

Rework the `KojiTargetResultOptions` to contain information about:
 - the built image
 - build log
 - osbuild manifest

Information about each file contains:
 - filename
 - checksum type
 - file checksum
 - file size

For now, only the built image information is set and consumed by the
worker.

Add custom JSON (un)marshaler for `KojiTargetResultOptions` to handle
backward compatibility when old version of worker or composer server
interact with each other. Cover them with unit tests.

Signed-off-by: Tomáš Hozza <thozza@redhat.com>
2023-09-27 12:39:28 +02:00

581 lines
18 KiB
Go

package v2_test
import (
"context"
"encoding/json"
"fmt"
"net/http"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"github.com/osbuild/images/pkg/distro/test_distro"
"github.com/osbuild/images/pkg/osbuild"
v2 "github.com/osbuild/osbuild-composer/internal/cloudapi/v2"
"github.com/osbuild/osbuild-composer/internal/target"
"github.com/osbuild/osbuild-composer/internal/test"
"github.com/osbuild/osbuild-composer/internal/worker"
"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
)
type jobResult struct {
Result interface{} `json:"result"`
}
func TestKojiCompose(t *testing.T) {
kojiServer, workerServer, _, cancel := newV2Server(t, t.TempDir(), []string{""}, false, false)
handler := kojiServer.Handler("/api/image-builder-composer/v2")
workerHandler := workerServer.Handler()
defer cancel()
type kojiCase struct {
initResult worker.KojiInitJobResult
buildResult worker.OSBuildJobResult
finalizeResult worker.KojiFinalizeJobResult
composeReplyCode int
composeReply string
composeStatus string
}
var cases = []kojiCase{
// #0
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
TargetResults: []*target.TargetResult{target.NewKojiTargetResult(&target.KojiTargetResultOptions{
Image: &target.KojiOutputInfo{
Filename: "test.img",
ChecksumType: target.ChecksumTypeMD5,
Checksum: "browns",
Size: 42,
},
})},
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "success"
},
"image_statuses": [
{
"status": "success"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "success"
}`,
},
// #1
{
initResult: worker.KojiInitJobResult{
KojiError: "failure",
},
buildResult: worker.OSBuildJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
TargetResults: []*target.TargetResult{target.NewKojiTargetResult(&target.KojiTargetResultOptions{
Image: &target.KojiOutputInfo{
Filename: "test.img",
ChecksumType: target.ChecksumTypeMD5,
Checksum: "browns",
Size: 42,
},
})},
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "failure"
},
"image_statuses": [
{
"status": "failure"
},
{
"status": "failure"
}
],
"koji_status": {},
"status": "failure"
}`,
},
// #2
{
initResult: worker.KojiInitJobResult{
JobResult: worker.JobResult{
JobError: clienterrors.WorkerClientError(clienterrors.ErrorKojiInit, "Koji init error", nil),
},
},
buildResult: worker.OSBuildJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
TargetResults: []*target.TargetResult{target.NewKojiTargetResult(&target.KojiTargetResultOptions{
Image: &target.KojiOutputInfo{
Filename: "test.img",
ChecksumType: target.ChecksumTypeMD5,
Checksum: "browns",
Size: 42,
},
})},
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "failure"
},
"image_statuses": [
{
"status": "failure"
},
{
"status": "failure"
}
],
"koji_status": {},
"status": "failure"
}`,
},
// #3
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
TargetResults: []*target.TargetResult{target.NewKojiTargetResult(&target.KojiTargetResultOptions{
Image: &target.KojiOutputInfo{
Filename: "test.img",
ChecksumType: target.ChecksumTypeMD5,
Checksum: "browns",
Size: 42,
},
})},
OSBuildOutput: &osbuild.Result{
Success: false,
},
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "failure"
},
"image_statuses": [
{
"status": "failure"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "failure"
}`,
},
// #4
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
TargetResults: []*target.TargetResult{target.NewKojiTargetResult(&target.KojiTargetResultOptions{
Image: &target.KojiOutputInfo{
Filename: "test.img",
ChecksumType: target.ChecksumTypeMD5,
Checksum: "browns",
Size: 42,
},
})},
OSBuildOutput: &osbuild.Result{
Success: true,
},
JobResult: worker.JobResult{
JobError: clienterrors.WorkerClientError(clienterrors.ErrorBuildJob, "Koji build error", nil),
},
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "failure",
"error": {
"id": 10,
"reason": "Koji build error"
}
},
"image_statuses": [
{
"status": "failure",
"error": {
"id": 10,
"reason": "Koji build error"
}
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "failure"
}`,
},
// #5
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
TargetResults: []*target.TargetResult{target.NewKojiTargetResult(&target.KojiTargetResultOptions{
Image: &target.KojiOutputInfo{
Filename: "test.img",
ChecksumType: target.ChecksumTypeMD5,
Checksum: "browns",
Size: 42,
},
})},
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
finalizeResult: worker.KojiFinalizeJobResult{
KojiError: "failure",
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "success"
},
"image_statuses": [
{
"status": "success"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "failure"
}`,
},
// #6
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
TargetResults: []*target.TargetResult{target.NewKojiTargetResult(&target.KojiTargetResultOptions{
Image: &target.KojiOutputInfo{
Filename: "test.img",
ChecksumType: target.ChecksumTypeMD5,
Checksum: "browns",
Size: 42,
},
})},
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
finalizeResult: worker.KojiFinalizeJobResult{
JobResult: worker.JobResult{
JobError: clienterrors.WorkerClientError(clienterrors.ErrorKojiFinalize, "Koji finalize error", nil),
},
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "success"
},
"image_statuses": [
{
"status": "success"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "failure"
}`,
},
}
emptyManifest := `{"version":"2","pipelines":[{"name":"build"},{"name":"os"}],"sources":{"org.osbuild.curl":{"items":{"sha256:e50ddb78a37f5851d1a5c37a4c77d59123153c156e628e064b9daa378f45a2fe":{"url":""}}}}}`
expectedManifests := `{"manifests":[` + emptyManifest + `,` + emptyManifest + `],"kind":"ComposeManifests"}`
for idx, c := range cases {
name, version, release := "foo", "1", "2"
t.Run(fmt.Sprintf("Test case #%d", idx), func(t *testing.T) {
composeRawReply := test.TestRouteWithReply(t, handler, false, "POST", "/api/image-builder-composer/v2/compose", fmt.Sprintf(`
{
"distribution":"%[1]s",
"image_requests": [
{
"architecture": "%[2]s",
"image_type": "%[3]s",
"repositories": [
{
"baseurl": "https://repo.example.com/"
}
]
},
{
"architecture": "%[2]s",
"image_type": "%[3]s",
"repositories": [
{
"baseurl": "https://repo.example.com/"
}
]
}
],
"koji": {
"server": "koji.example.com",
"name":"%[4]s",
"version":"%[5]s",
"release":"%[6]s",
"task_id": 42
}
}`, test_distro.TestDistroName, test_distro.TestArch3Name, string(v2.ImageTypesGuestImage), name, version, release),
c.composeReplyCode, c.composeReply, "id", "operation_id")
// determine the compose ID from the reply
var composeReply v2.ComposeId
err := json.Unmarshal(composeRawReply, &composeReply)
require.NoError(t, err)
composeId, err := uuid.Parse(composeReply.Id)
require.NoError(t, err)
// handle koji-init
_, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{worker.JobTypeKojiInit}, []string{""})
require.NoError(t, err)
require.Equal(t, worker.JobTypeKojiInit, jobType)
var initJob worker.KojiInitJob
err = json.Unmarshal(rawJob, &initJob)
require.NoError(t, err)
require.Equal(t, "koji.example.com", initJob.Server)
require.Equal(t, "foo", initJob.Name)
require.Equal(t, "1", initJob.Version)
require.Equal(t, "2", initJob.Release)
initJobResult, err := json.Marshal(&jobResult{Result: c.initResult})
require.NoError(t, err)
test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), string(initJobResult), http.StatusOK,
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
// Finishing of the goroutine handling the manifest job is not deterministic and as a result, we may get
// the second osbuild job first.
// The build jobs ID is determined from the dependencies of the koji-finalize job dependencies.
finalizeInfo, err := workerServer.KojiFinalizeJobInfo(composeId, &worker.KojiFinalizeJobResult{})
require.NoError(t, err)
buildJobIDs := finalizeInfo.Deps[1:]
require.Len(t, buildJobIDs, 2)
// handle build jobs
for i := 0; i < len(buildJobIDs); i++ {
jobID, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{worker.JobTypeOSBuild}, []string{""})
require.NoError(t, err)
require.Equal(t, worker.JobTypeOSBuild, jobType)
var osbuildJob worker.OSBuildJob
err = json.Unmarshal(rawJob, &osbuildJob)
require.NoError(t, err)
jobTarget := osbuildJob.Targets[0].Options.(*target.KojiTargetOptions)
require.Equal(t, "koji.example.com", jobTarget.Server)
require.Equal(t, "test.img", osbuildJob.Targets[0].OsbuildArtifact.ExportFilename)
require.Equal(t, fmt.Sprintf("%s-%s-%s.%s.img", name, version, release, test_distro.TestArch3Name),
osbuildJob.Targets[0].ImageName)
require.NotEmpty(t, jobTarget.UploadDirectory)
var buildJobResult string
switch jobID {
// use the build job result from the test case only for the first job
case buildJobIDs[0]:
buildJobResultBytes, err := json.Marshal(&jobResult{Result: c.buildResult})
require.NoError(t, err)
buildJobResult = string(buildJobResultBytes)
default:
buildJobResult = fmt.Sprintf(`{
"result": {
"arch": "%s",
"host_os": "%s",
"image_hash": "browns",
"image_size": 42,
"osbuild_output": {
"success": true
}
}
}`, test_distro.TestArch3Name, test_distro.TestDistroName)
}
test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), buildJobResult, http.StatusOK,
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
}
// handle koji-finalize
finalizeID, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{worker.JobTypeKojiFinalize}, []string{""})
require.NoError(t, err)
require.Equal(t, worker.JobTypeKojiFinalize, jobType)
var kojiFinalizeJob worker.KojiFinalizeJob
err = json.Unmarshal(rawJob, &kojiFinalizeJob)
require.NoError(t, err)
require.Equal(t, "koji.example.com", kojiFinalizeJob.Server)
require.Equal(t, "1", kojiFinalizeJob.Version)
require.Equal(t, "2", kojiFinalizeJob.Release)
require.ElementsMatch(t, []string{
fmt.Sprintf("foo-1-2.%s.img", test_distro.TestArch3Name),
fmt.Sprintf("foo-1-2.%s.img", test_distro.TestArch3Name),
}, kojiFinalizeJob.KojiFilenames)
require.NotEmpty(t, kojiFinalizeJob.KojiDirectory)
finalizeResult, err := json.Marshal(&jobResult{Result: c.finalizeResult})
require.NoError(t, err)
test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), string(finalizeResult), http.StatusOK,
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
// get the status
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v", finalizeID), ``, http.StatusOK, c.composeStatus, `href`, `id`)
// get the manifests
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v/manifests", finalizeID), ``, http.StatusOK, expectedManifests, `href`, `id`)
// get the logs
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v/logs", finalizeID), ``, http.StatusOK, `{"kind":"ComposeLogs"}`, `koji`, `image_builds`, `href`, `id`)
})
}
}
func TestKojiJobTypeValidation(t *testing.T) {
server, workers, _, cancel := newV2Server(t, t.TempDir(), []string{""}, false, false)
handler := server.Handler("/api/image-builder-composer/v2")
defer cancel()
// Enqueue a compose job with N images (+ an Init and a Finalize job)
// Enqueuing them manually gives us access to the job IDs to use in
// requests.
// TODO: set to 4
nImages := 1
initJob := worker.KojiInitJob{
Server: "test-server",
Name: "test-job",
Version: "42",
Release: "1",
}
initID, err := workers.EnqueueKojiInit(&initJob, "")
require.NoError(t, err)
manifest, err := json.Marshal(osbuild.Manifest{})
require.NoErrorf(t, err, "error marshalling empty Manifest to JSON")
buildJobs := make([]worker.OSBuildJob, nImages)
buildJobIDs := make([]uuid.UUID, nImages)
filenames := make([]string, nImages)
for idx := 0; idx < nImages; idx++ {
kojiTarget := target.NewKojiTarget(&target.KojiTargetOptions{
Server: "test-server",
UploadDirectory: "koji-server-test-dir",
})
kojiTarget.OsbuildArtifact.ExportFilename = "test.img"
kojiTarget.ImageName = fmt.Sprintf("image-file-%04d", idx)
buildJob := worker.OSBuildJob{
Targets: []*target.Target{kojiTarget},
// Add an empty manifest as a static job argument to make the test pass.
// Becasue of a bug in the API, the test was passing even without
// any manifest being attached to the job (static or dynamic).
// In reality, cloudapi never adds the manifest as a static job argument.
// TODO: use dependent depsolve and manifests jobs instead
Manifest: manifest,
}
buildID, err := workers.EnqueueOSBuildAsDependency(fmt.Sprintf("fake-arch-%d", idx), &buildJob, []uuid.UUID{initID}, "")
require.NoError(t, err)
buildJobs[idx] = buildJob
buildJobIDs[idx] = buildID
filenames[idx] = kojiTarget.OsbuildArtifact.ExportFilename
}
finalizeJob := worker.KojiFinalizeJob{
Server: "test-server",
Name: "test-job",
Version: "42",
Release: "1",
KojiFilenames: filenames,
KojiDirectory: "koji-server-test-dir",
TaskID: 0,
StartTime: uint64(time.Now().Unix()),
}
finalizeID, err := workers.EnqueueKojiFinalize(&finalizeJob, initID, buildJobIDs, "")
require.NoError(t, err)
// ----- Jobs queued - Test API endpoints (status, manifests, logs) ----- //
t.Logf("%q job ID: %s", worker.JobTypeKojiInit, initID)
t.Logf("%q job ID: %s", worker.JobTypeKojiFinalize, finalizeID)
t.Logf("%q job IDs: %v", worker.JobTypeOSBuild, buildJobIDs)
for _, path := range []string{"", "/manifests", "/logs"} {
// should return OK - actual result should be tested elsewhere
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%s%s", finalizeID, path), ``, http.StatusOK, "*")
// The other IDs should fail
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%s%s", initID, path), ``, http.StatusNotFound, `{"code":"IMAGE-BUILDER-COMPOSER-26", "details": "", "href":"/api/image-builder-composer/v2/errors/26","id":"26","kind":"Error","reason":"Job with given id has an invalid type"}`, `operation_id`)
for _, buildID := range buildJobIDs {
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%s%s", buildID, path), ``, http.StatusOK, "*")
}
badID := uuid.New()
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%s%s", badID, path), ``, http.StatusNotFound, `{"code":"IMAGE-BUILDER-COMPOSER-15", "details": "", "href":"/api/image-builder-composer/v2/errors/15","id":"15","kind":"Error","reason":"Compose with given id not found"}`, `operation_id`)
}
}