cloudapi/v2: add koji-specific tests

These are based on the existing tests for the koji API, making sure the coverage is the same.
This commit is contained in:
Tom Gundersen 2022-01-28 14:57:14 +00:00
parent a3f8304ae8
commit 11e2ae4528
5 changed files with 707 additions and 2 deletions

View file

@ -0,0 +1,25 @@
package v2
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestSplitExtension(t *testing.T) {
tests := []struct {
filename string
extension string
}{
{filename: "image.qcow2", extension: ".qcow2"},
{filename: "image.tar.gz", extension: ".tar.gz"},
{filename: "", extension: ""},
{filename: ".htaccess", extension: ""},
{filename: ".weirdfile.txt", extension: ".txt"},
}
for _, tt := range tests {
t.Run(tt.filename, func(t *testing.T) {
require.Equal(t, tt.extension, splitExtension(tt.filename))
})
}
}

View file

@ -0,0 +1,564 @@
package v2_test
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
"time"
"github.com/google/uuid"
v2 "github.com/osbuild/osbuild-composer/internal/cloudapi/v2"
"github.com/osbuild/osbuild-composer/internal/distro/test_distro"
"github.com/osbuild/osbuild-composer/internal/kojiapi/api"
osbuild "github.com/osbuild/osbuild-composer/internal/osbuild2"
"github.com/osbuild/osbuild-composer/internal/test"
"github.com/osbuild/osbuild-composer/internal/worker"
"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
"github.com/stretchr/testify/require"
)
type jobResult struct {
Result interface{} `json:"result"`
}
func TestKojiCompose(t *testing.T) {
dir, err := ioutil.TempDir("", "osbuild-composer-test-api-v2-")
require.NoError(t, err)
defer os.RemoveAll(dir)
kojiServer, workerServer, cancel := newV2Server(t, dir)
handler := kojiServer.Handler("/api/image-builder-composer/v2")
workerHandler := workerServer.Handler()
defer cancel()
type kojiCase struct {
initResult worker.KojiInitJobResult
buildResult worker.OSBuildKojiJobResult
finalizeResult worker.KojiFinalizeJobResult
composeReplyCode int
composeReply string
composeStatus string
}
var cases = []kojiCase{
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildKojiJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
ImageHash: "browns",
ImageSize: 42,
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "success"
},
"image_statuses": [
{
"status": "success"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "success"
}`,
},
{
initResult: worker.KojiInitJobResult{
KojiError: "failure",
},
buildResult: worker.OSBuildKojiJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
ImageHash: "browns",
ImageSize: 42,
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "failure"
},
"image_statuses": [
{
"status": "failure"
},
{
"status": "failure"
}
],
"koji_status": {},
"status": "failure"
}`,
},
{
initResult: worker.KojiInitJobResult{
JobResult: worker.JobResult{
JobError: clienterrors.WorkerClientError(clienterrors.ErrorKojiInit, "Koji init error"),
},
},
buildResult: worker.OSBuildKojiJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
ImageHash: "browns",
ImageSize: 42,
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
composeReplyCode: http.StatusCreated,
composeReply: `{"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"}`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "failure"
},
"image_statuses": [
{
"status": "failure"
},
{
"status": "failure"
}
],
"koji_status": {},
"status": "failure"
}`,
},
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildKojiJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
ImageHash: "browns",
ImageSize: 42,
OSBuildOutput: &osbuild.Result{
Success: false,
},
},
composeReplyCode: http.StatusCreated,
composeReply: `"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "failure"
},
"image_statuses": [
{
"status": "failure"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "failure"
}`,
},
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildKojiJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
ImageHash: "browns",
ImageSize: 42,
OSBuildOutput: &osbuild.Result{
Success: true,
},
KojiError: "failure",
},
composeReplyCode: http.StatusCreated,
composeReply: `"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "failure"
},
"image_statuses": [
{
"status": "failure"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "failure"
}`,
},
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildKojiJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
ImageHash: "browns",
ImageSize: 42,
OSBuildOutput: &osbuild.Result{
Success: true,
},
JobResult: worker.JobResult{
JobError: clienterrors.WorkerClientError(clienterrors.ErrorBuildJob, "Koji build error"),
},
},
composeReplyCode: http.StatusCreated,
composeReply: `"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "failure"
},
"image_statuses": [
{
"status": "failure"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "failure"
}`,
},
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildKojiJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
ImageHash: "browns",
ImageSize: 42,
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
finalizeResult: worker.KojiFinalizeJobResult{
KojiError: "failure",
},
composeReplyCode: http.StatusCreated,
composeReply: `"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "success"
},
"image_statuses": [
{
"status": "success"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "failure"
}`,
},
{
initResult: worker.KojiInitJobResult{
BuildID: 42,
Token: `"foobar"`,
},
buildResult: worker.OSBuildKojiJobResult{
Arch: test_distro.TestArchName,
HostOS: test_distro.TestDistroName,
ImageHash: "browns",
ImageSize: 42,
OSBuildOutput: &osbuild.Result{
Success: true,
},
},
finalizeResult: worker.KojiFinalizeJobResult{
JobResult: worker.JobResult{
JobError: clienterrors.WorkerClientError(clienterrors.ErrorKojiFinalize, "Koji finalize error"),
},
},
composeReplyCode: http.StatusCreated,
composeReply: `"href":"/api/image-builder-composer/v2/compose", "kind":"ComposeId"`,
composeStatus: `{
"kind": "ComposeStatus",
"image_status": {
"status": "success"
},
"image_statuses": [
{
"status": "success"
},
{
"status": "success"
}
],
"koji_status": {
"build_id": 42
},
"status": "failure"
}`,
},
}
for _, c := range cases[2:3] {
test.TestRoute(t, handler, false, "POST", "/api/image-builder-composer/v2/compose", fmt.Sprintf(`
{
"distribution":"%[1]s",
"image_requests": [
{
"architecture": "%[2]s",
"image_type": "%[3]s",
"repositories": [
{
"baseurl": "https://repo.example.com/"
}
]
},
{
"architecture": "%[2]s",
"image_type": "%[3]s",
"repositories": [
{
"baseurl": "https://repo.example.com/"
}
]
}
],
"koji": {
"server": "koji.example.com",
"name":"foo",
"version":"1",
"release":"2"
}
}`, test_distro.TestDistroName, test_distro.TestArch3Name, string(v2.ImageTypesGuestImage)),
c.composeReplyCode, c.composeReply, "id", "operation_id")
// handle koji-init
_, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"koji-init"})
require.NoError(t, err)
require.Equal(t, "koji-init", jobType)
var initJob worker.KojiInitJob
err = json.Unmarshal(rawJob, &initJob)
require.NoError(t, err)
require.Equal(t, "koji.example.com", initJob.Server)
require.Equal(t, "foo", initJob.Name)
require.Equal(t, "1", initJob.Version)
require.Equal(t, "2", initJob.Release)
initJobResult, err := json.Marshal(&jobResult{Result: c.initResult})
require.NoError(t, err)
test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), string(initJobResult), http.StatusOK,
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
// handle osbuild-koji #1
_, token, jobType, rawJob, _, err = workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild-koji"})
require.NoError(t, err)
require.Equal(t, "osbuild-koji", jobType)
var osbuildJob worker.OSBuildKojiJob
err = json.Unmarshal(rawJob, &osbuildJob)
require.NoError(t, err)
require.Equal(t, "koji.example.com", osbuildJob.KojiServer)
require.Equal(t, "test.img", osbuildJob.ImageName)
require.NotEmpty(t, osbuildJob.KojiDirectory)
buildJobResult, err := json.Marshal(&jobResult{Result: c.buildResult})
require.NoError(t, err)
test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), string(buildJobResult), http.StatusOK,
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
// handle osbuild-koji #2
_, token, jobType, rawJob, _, err = workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"osbuild-koji"})
require.NoError(t, err)
require.Equal(t, "osbuild-koji", jobType)
err = json.Unmarshal(rawJob, &osbuildJob)
require.NoError(t, err)
require.Equal(t, "koji.example.com", osbuildJob.KojiServer)
require.Equal(t, "test.img", osbuildJob.ImageName)
require.NotEmpty(t, osbuildJob.KojiDirectory)
test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), fmt.Sprintf(`{
"result": {
"arch": "%s",
"host_os": "%s",
"image_hash": "browns",
"image_size": 42,
"osbuild_output": {
"success": true
}
}
}`, test_distro.TestArch3Name, test_distro.TestDistroName), http.StatusOK,
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
// handle koji-finalize
finalizeID, token, jobType, rawJob, _, err := workerServer.RequestJob(context.Background(), test_distro.TestArch3Name, []string{"koji-finalize"})
require.NoError(t, err)
require.Equal(t, "koji-finalize", jobType)
var kojiFinalizeJob worker.KojiFinalizeJob
err = json.Unmarshal(rawJob, &kojiFinalizeJob)
require.NoError(t, err)
require.Equal(t, "koji.example.com", kojiFinalizeJob.Server)
require.Equal(t, "1", kojiFinalizeJob.Version)
require.Equal(t, "2", kojiFinalizeJob.Release)
require.ElementsMatch(t, []string{
fmt.Sprintf("foo-1-2.%s.img", test_distro.TestArch3Name),
fmt.Sprintf("foo-1-2.%s.img", test_distro.TestArch3Name),
}, kojiFinalizeJob.KojiFilenames)
require.NotEmpty(t, kojiFinalizeJob.KojiDirectory)
finalizeResult, err := json.Marshal(&jobResult{Result: c.finalizeResult})
require.NoError(t, err)
test.TestRoute(t, workerHandler, false, "PATCH", fmt.Sprintf("/api/worker/v1/jobs/%v", token), string(finalizeResult), http.StatusOK,
fmt.Sprintf(`{"href":"/api/worker/v1/jobs/%v","id":"%v","kind":"UpdateJobResponse"}`, token, token))
// get the status
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v", finalizeID), ``, http.StatusOK, c.composeStatus, `href`, `id`)
// get the manifests
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v/manifests", finalizeID), ``, http.StatusOK, `{"manifests":[null,null],"kind":"ComposeManifests"}`, `href`, `id`)
// get the logs
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v/logs", finalizeID), ``, http.StatusOK, `{"kind":"ComposeLogs"}`, `koji`, `image_builds`, `href`, `id`)
}
}
func TestKojiRequest(t *testing.T) {
dir, err := ioutil.TempDir("", "osbuild-composer-test-api-v2-")
require.NoError(t, err)
defer os.RemoveAll(dir)
server, _, cancel := newV2Server(t, dir)
handler := server.Handler("/api/image-builder-composer/v2")
defer cancel()
// Make request to an invalid route
req := httptest.NewRequest("GET", "/invalidroute", nil)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
resp := rec.Result()
var status api.Status
err = json.NewDecoder(resp.Body).Decode(&status)
require.NoError(t, err)
require.Equal(t, http.StatusNotFound, resp.StatusCode)
// Trigger an error 400 code
req = httptest.NewRequest("GET", "/api/image-builder-composer/v2/composes/badid", nil)
rec = httptest.NewRecorder()
handler.ServeHTTP(rec, req)
resp = rec.Result()
err = json.NewDecoder(resp.Body).Decode(&status)
require.NoError(t, err)
require.Equal(t, http.StatusBadRequest, resp.StatusCode)
}
func TestKojiJobTypeValidation(t *testing.T) {
dir, err := ioutil.TempDir("", "osbuild-composer-test-api-v2-")
require.NoError(t, err)
defer os.RemoveAll(dir)
server, workers, cancel := newV2Server(t, dir)
handler := server.Handler("/api/image-builder-composer/v2")
defer cancel()
// Enqueue a compose job with N images (+ an Init and a Finalize job)
// Enqueuing them manually gives us access to the job IDs to use in
// requests.
// TODO: set to 4
nImages := 1
initJob := worker.KojiInitJob{
Server: "test-server",
Name: "test-job",
Version: "42",
Release: "1",
}
initID, err := workers.EnqueueKojiInit(&initJob)
require.NoError(t, err)
buildJobs := make([]worker.OSBuildKojiJob, nImages)
buildJobIDs := make([]uuid.UUID, nImages)
filenames := make([]string, nImages)
for idx := 0; idx < nImages; idx++ {
fname := fmt.Sprintf("image-file-%04d", idx)
buildJob := worker.OSBuildKojiJob{
ImageName: fmt.Sprintf("build-job-%04d", idx),
KojiServer: "test-server",
KojiDirectory: "koji-server-test-dir",
KojiFilename: fname,
}
buildID, err := workers.EnqueueOSBuildKoji(fmt.Sprintf("fake-arch-%d", idx), &buildJob, initID)
require.NoError(t, err)
buildJobs[idx] = buildJob
buildJobIDs[idx] = buildID
filenames[idx] = fname
}
finalizeJob := worker.KojiFinalizeJob{
Server: "test-server",
Name: "test-job",
Version: "42",
Release: "1",
KojiFilenames: filenames,
KojiDirectory: "koji-server-test-dir",
TaskID: 0,
StartTime: uint64(time.Now().Unix()),
}
finalizeID, err := workers.EnqueueKojiFinalize(&finalizeJob, initID, buildJobIDs)
require.NoError(t, err)
// ----- Jobs queued - Test API endpoints (status, manifests, logs) ----- //
for _, path := range []string{"", "/manifests", "/logs"} {
// should return OK - actual result should be tested elsewhere
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%s%s", finalizeID, path), ``, http.StatusOK, "*")
// The other IDs should fail
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%s%s", initID, path), ``, http.StatusNotFound, `{"code":"IMAGE-BUILDER-COMPOSER-26","href":"/api/image-builder-composer/v2/errors/26","id":"26","kind":"Error","reason":"Requested job has invalid type"}`, `operation_id`)
for _, buildID := range buildJobIDs {
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%s%s", buildID, path), ``, http.StatusNotFound, `{"code":"IMAGE-BUILDER-COMPOSER-26","href":"/api/image-builder-composer/v2/errors/26","id":"26","kind":"Error","reason":"Requested job has invalid type"}`, `operation_id`)
}
badID := uuid.New()
test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%s%s", badID, path), ``, http.StatusNotFound, `{"code":"IMAGE-BUILDER-COMPOSER-15","href":"/api/image-builder-composer/v2/errors/15","id":"15","kind":"Error","reason":"Compose with given id not found"}`, `operation_id`)
}
}

View file

@ -220,6 +220,7 @@ install -m 0755 -vp tools/gen-ssh.sh %{buildroot}%
install -m 0755 -vp tools/image-info %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/run-koji-container.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/koji-compose.py %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/koji-compose-v2.py %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/libvirt_test.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/set-env-variables.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/
install -m 0755 -vp tools/test-case-generators/generate-test-cases %{buildroot}%{_libexecdir}/osbuild-composer-test/

View file

@ -57,8 +57,13 @@ if [[ "$DISTRO_CODE" == rhel-8* ]]; then
DISTRO_CODE=rhel-86
fi
greenprint "Pushing compose to Koji"
sudo /usr/libexec/osbuild-composer-test/koji-compose.py "$DISTRO_CODE" "${ARCH}"
if [ "${COMPOSER_API:=true}" == "true" ]; then
greenprint "Pushing compose to Koji (/api/image-builder-comoser/v2/"
sudo /usr/libexec/osbuild-composer-test/koji-compose-v2.py "$DISTRO_CODE" "${ARCH}"
else
greenprint "Pushing compose to Koji (/api/comoser-koji/v1/"
sudo /usr/libexec/osbuild-composer-test/koji-compose.py "$DISTRO_CODE" "${ARCH}"
fi
greenprint "Show Koji task"
koji --server=http://localhost:8080/kojihub taskinfo 1

110
tools/koji-compose-v2.py Executable file
View file

@ -0,0 +1,110 @@
#!/usr/bin/python3
import json
import sys
import time
import requests
# Composer API for Koji uses a slightly different repository format
# that osbuild-composer does in /usr/share/osbuild-composer/repositories.
#
# This function does the conversion.
def composer_repository_to_koji_repository(repository):
koji_repository = {
"baseurl": repository["baseurl"]
}
if repository.get("check_gpg", False):
koji_repository["gpgkey"] = repository["gpgkey"]
return koji_repository
def compose_request(distro, koji, arch):
with open(f"/usr/share/tests/osbuild-composer/repositories/{distro}.json") as f:
test_repositories = json.load(f)
repositories = [composer_repository_to_koji_repository(repo) for repo in test_repositories[arch]]
image_requests = [
{
"architecture": "x86_64",
"image_type": "guest-image",
"repositories": repositories
},
{
"architecture": "x86_64",
"image_type": "aws",
"repositories": repositories
}
]
req = {
"distribution": distro,
"koji": {
"server": koji,
"task_id": 1,
"name": "name",
"version": "version",
"release": "release",
},
"image_requests": image_requests
}
return req
def main(distro, arch):
cr = compose_request(distro, "https://localhost:4343/kojihub", arch)
print(json.dumps(cr))
r = requests.post("https://localhost/api/image-builder-composer/v2/compose", json=cr,
cert=("/etc/osbuild-composer/worker-crt.pem", "/etc/osbuild-composer/worker-key.pem"),
verify="/etc/osbuild-composer/ca-crt.pem")
if r.status_code != 201:
print("Failed to create compose")
print(r.text)
sys.exit(1)
print(r.text)
compose_id = r.json()["id"]
while True:
r = requests.get(f"https://localhost/api/image-builder-composer/v2/composes/{compose_id}",
cert=("/etc/osbuild-composer/worker-crt.pem", "/etc/osbuild-composer/worker-key.pem"),
verify="/etc/osbuild-composer/ca-crt.pem")
if r.status_code != 200:
print("Failed to get compose status")
print(r.text)
sys.exit(1)
status = r.json()["status"]
print(status)
if status == "success":
print("Compose worked!")
print(r.text)
break
elif status == "failure":
print("compose failed!")
print(r.text)
sys.exit(1)
elif status != "pending" and status != "running":
print(f"unexpected status: {status}")
print(r.text)
sys.exit(1)
time.sleep(10)
r = requests.get(f"https://localhost/api/image-builder-composer/v2/composes/{compose_id}/logs",
cert=("/etc/osbuild-composer/worker-crt.pem", "/etc/osbuild-composer/worker-key.pem"),
verify="/etc/osbuild-composer/ca-crt.pem")
logs = r.json()
assert "image_builds" in logs
assert type(logs["image_builds"]) == list
assert len(logs["image_builds"]) == len(cr["image_requests"])
if __name__ == "__main__":
if len(sys.argv) != 3:
print(f"usage: {sys.argv[0]} DISTRO ARCH", file=sys.stderr)
sys.exit(1)
main(sys.argv[1], sys.argv[2])