diff --git a/internal/cloudapi/v2/v2_koji_test.go b/internal/cloudapi/v2/v2_koji_test.go index b9ad65f5c..44661e51a 100644 --- a/internal/cloudapi/v2/v2_koji_test.go +++ b/internal/cloudapi/v2/v2_koji_test.go @@ -323,6 +323,8 @@ func TestKojiCompose(t *testing.T) { }, } + emptyManifest := `{"version":"2","pipelines":[{"name":"build"},{"name":"os"}],"sources":{"org.osbuild.curl":{"items":{"":{"url":""}}}}}` + expectedManifests := `{"manifests":[` + emptyManifest + `,` + emptyManifest + `],"kind":"ComposeManifests"}` for idx, c := range cases { name, version, release := "foo", "1", "2" t.Run(fmt.Sprintf("Test case #%d", idx), func(t *testing.T) { @@ -459,7 +461,7 @@ func TestKojiCompose(t *testing.T) { test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v", finalizeID), ``, http.StatusOK, c.composeStatus, `href`, `id`) // get the manifests - test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v/manifests", finalizeID), ``, http.StatusOK, `{"manifests":[{"version":"2","pipelines":[],"sources":{}},{"version":"2","pipelines":[],"sources":{}}],"kind":"ComposeManifests"}`, `href`, `id`) + test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v/manifests", finalizeID), ``, http.StatusOK, expectedManifests, `href`, `id`) // get the logs test.TestRoute(t, handler, false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v/logs", finalizeID), ``, http.StatusOK, `{"kind":"ComposeLogs"}`, `koji`, `image_builds`, `href`, `id`) diff --git a/internal/cloudapi/v2/v2_test.go b/internal/cloudapi/v2/v2_test.go index e9a058e7e..dc2d420f3 100644 --- a/internal/cloudapi/v2/v2_test.go +++ b/internal/cloudapi/v2/v2_test.go @@ -585,19 +585,16 @@ func TestComposeStatusSuccess(t *testing.T) { ] }`, jobId, jobId)) + emptyManifest := `{"version":"2","pipelines":[{"name":"build"},{"name":"os"}],"sources":{"org.osbuild.curl":{"items":{"":{"url":""}}}}}` test.TestRoute(t, srv.Handler("/api/image-builder-composer/v2"), false, "GET", fmt.Sprintf("/api/image-builder-composer/v2/composes/%v/manifests", jobId), ``, http.StatusOK, fmt.Sprintf(` { "href": "/api/image-builder-composer/v2/composes/%v/manifests", "id": "%v", "kind": "ComposeManifests", "manifests": [ - { - "version": "2", - "pipelines": [], - "sources": {} - } + %s ] - }`, jobId, jobId), "details") + }`, jobId, jobId, emptyManifest), "details") } func TestComposeStatusFailure(t *testing.T) { diff --git a/internal/ostree/mock_ostree_repo/mock_repo.go b/internal/ostree/mock_ostree_repo/mock_repo.go index 61b22231c..19c38449f 100644 --- a/internal/ostree/mock_ostree_repo/mock_repo.go +++ b/internal/ostree/mock_ostree_repo/mock_repo.go @@ -1,6 +1,7 @@ package mock_ostree_repo import ( + "crypto/sha256" "fmt" "net/http" "net/http/httptest" @@ -23,15 +24,17 @@ func Setup(ref string) *OSTreeTestRepo { repo.OSTreeRef = ref mux := http.NewServeMux() + repo.Server = httptest.NewServer(mux) + + checksum := fmt.Sprintf("%x", sha256.Sum256([]byte(repo.Server.URL+ref))) + fmt.Printf("Creating repo with %s %s %s\n", ref, repo.Server.URL, checksum) mux.HandleFunc("/refs/heads/"+ref, func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, "02604b2da6e954bd34b8b82a835e5a77d2b60ffa") + fmt.Fprint(w, checksum) }) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { // catch-all handler, return 404 http.NotFound(w, r) }) - repo.Server = httptest.NewServer(mux) - return repo } diff --git a/internal/weldr/api.go b/internal/weldr/api.go index 8a579fa3c..92ab30fa5 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "crypto/rand" + "crypto/sha256" "encoding/json" "errors" errors_package "errors" @@ -2342,7 +2343,7 @@ func (api *API) resolveOSTreeCommits(sourceSpecs map[string][]ostree.SourceSpec, for idx, source := range sources { var ref, checksum string if test { - checksum = "02604b2da6e954bd34b8b82a835e5a77d2b60ffa" + checksum = fmt.Sprintf("%x", sha256.Sum256([]byte(source.URL+source.Ref))) ref = source.Ref } else { var err error diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go index c73b05bbf..7d0af5c6c 100644 --- a/internal/weldr/api_test.go +++ b/internal/weldr/api_test.go @@ -25,6 +25,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/dnfjson" dnfjson_mock "github.com/osbuild/osbuild-composer/internal/mocks/dnfjson" rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd" + "github.com/osbuild/osbuild-composer/internal/ostree" "github.com/osbuild/osbuild-composer/internal/ostree/mock_ostree_repo" "github.com/osbuild/osbuild-composer/internal/reporegistry" "github.com/osbuild/osbuild-composer/internal/rpmmd" @@ -879,6 +880,13 @@ func TestNewBlueprintsUndo(t *testing.T) { } func TestCompose(t *testing.T) { + // create two ostree repos, one to serve the default test_distro ref (for fallback tests) and one to serve a custom ref + ostreeRepoDefault := mock_ostree_repo.Setup(test_distro.New().OSTreeRef()) + defer ostreeRepoDefault.TearDown() + otherRef := "some/other/ref" + ostreeRepoOther := mock_ostree_repo.Setup(otherRef) + defer ostreeRepoOther.TearDown() + arch, err := test_distro.New().GetArch(test_distro.TestArchName) require.NoError(t, err) imgType, err := arch.GetImageType(test_distro.TestImageTypeName) @@ -886,15 +894,20 @@ func TestCompose(t *testing.T) { manifest, _, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, 0) require.NoError(t, err) - mf, err := manifest.Serialize(nil, nil, nil) + rPkgs, rContainers, rCommits := test_distro.ResolveContent(manifest.Content.PackageSets, manifest.Content.Containers, manifest.Content.OSTreeCommits) + + mf, err := manifest.Serialize(rPkgs, rContainers, rCommits) require.NoError(t, err) ostreeImgType, err := arch.GetImageType(test_distro.TestImageTypeOSTree) require.NoError(t, err) - ostreeManifest, _, err := ostreeImgType.Manifest(nil, distro.ImageOptions{}, nil, 0) + ostreeOptions := ostree.ImageOptions{URL: ostreeRepoDefault.Server.URL} + ostreeManifest, _, err := ostreeImgType.Manifest(nil, distro.ImageOptions{OSTree: &ostreeOptions}, nil, 0) require.NoError(t, err) - omf, err := ostreeManifest.Serialize(nil, nil, nil) + rPkgs, rContainers, rCommits = test_distro.ResolveContent(ostreeManifest.Content.PackageSets, ostreeManifest.Content.Containers, ostreeManifest.Content.OSTreeCommits) + + omf, err := ostreeManifest.Serialize(rPkgs, rContainers, rCommits) require.NoError(t, err) expectedComposeLocal := &store.Compose{ @@ -996,6 +1009,42 @@ func TestCompose(t *testing.T) { Packages: dnfjson_mock.BaseDeps(), } + ostreeOptionsOther := ostree.ImageOptions{ImageRef: otherRef, URL: ostreeRepoOther.Server.URL} + ostreeManifestOther, _, err := ostreeImgType.Manifest(nil, distro.ImageOptions{OSTree: &ostreeOptionsOther}, nil, 0) + require.NoError(t, err) + + rPkgs, rContainers, rCommits = test_distro.ResolveContent(ostreeManifestOther.Content.PackageSets, ostreeManifestOther.Content.Containers, ostreeManifestOther.Content.OSTreeCommits) + + omfo, err := ostreeManifest.Serialize(rPkgs, rContainers, rCommits) + require.NoError(t, err) + expectedComposeOSTreeOther := &store.Compose{ + Blueprint: &blueprint.Blueprint{ + Name: "test", + Version: "0.0.0", + Packages: []blueprint.Package{}, + Modules: []blueprint.Package{}, + Groups: []blueprint.Group{}, + Customizations: nil, + }, + ImageBuild: store.ImageBuild{ + QueueStatus: common.IBWaiting, + ImageType: ostreeImgType, + Manifest: omfo, + Targets: []*target.Target{ + { + ImageName: ostreeImgType.Filename(), + OsbuildArtifact: target.OsbuildArtifact{ + ExportFilename: ostreeImgType.Filename(), + ExportName: ostreeImgType.Exports()[0], + }, + Name: target.TargetNameWorkerServer, + Options: &target.WorkerServerTargetOptions{}, + }, + }, + }, + Packages: dnfjson_mock.BaseDeps(), + } + // For 2nd distribution arch2, err := test_distro.New2().GetArch(test_distro.TestArchName) require.NoError(t, err) @@ -1004,7 +1053,8 @@ func TestCompose(t *testing.T) { manifest2, _, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, 0) require.NoError(t, err) - mf2, err := manifest2.Serialize(nil, nil, nil) + rPkgs, rContainers, rCommits = test_distro.ResolveContent(manifest2.Content.PackageSets, manifest2.Content.Containers, manifest2.Content.OSTreeCommits) + mf2, err := manifest2.Serialize(rPkgs, rContainers, rCommits) require.NoError(t, err) expectedComposeGoodDistro := &store.Compose{ @@ -1036,13 +1086,7 @@ func TestCompose(t *testing.T) { Packages: dnfjson_mock.BaseDeps(), } - // create two ostree repos, one to serve the default test_distro ref (for fallback tests) and one to serve a custom ref - ostreeRepoDefault := mock_ostree_repo.Setup(test_distro.New().OSTreeRef()) - defer ostreeRepoDefault.TearDown() - ostreeRepoOther := mock_ostree_repo.Setup("some/other/ref") - defer ostreeRepoOther.TearDown() - - var cases = []struct { + var cases = map[string]struct { External bool Method string Path string @@ -1052,7 +1096,7 @@ func TestCompose(t *testing.T) { ExpectedCompose *store.Compose IgnoreFields []string }{ - { + "bad-request": { true, "POST", "/api/v0/compose", @@ -1062,7 +1106,7 @@ func TestCompose(t *testing.T) { nil, []string{"build_id", "warnings"}, }, - { + "local": { false, "POST", "/api/v0/compose", @@ -1072,7 +1116,7 @@ func TestCompose(t *testing.T) { expectedComposeLocal, []string{"build_id", "warnings"}, }, - { + "aws": { false, "POST", "/api/v1/compose", @@ -1082,7 +1126,7 @@ func TestCompose(t *testing.T) { expectedComposeLocalAndAws, []string{"build_id", "warnings"}, }, - { + "good-distro": { false, "POST", "/api/v1/compose", @@ -1092,7 +1136,7 @@ func TestCompose(t *testing.T) { expectedComposeGoodDistro, []string{"build_id", "warnings"}, }, - { + "unknown-distro": { false, "POST", "/api/v1/compose", @@ -1102,7 +1146,7 @@ func TestCompose(t *testing.T) { nil, []string{"build_id", "warnings"}, }, - { + "imaginary": { false, "POST", "/api/v1/compose", @@ -1115,7 +1159,7 @@ func TestCompose(t *testing.T) { // === OSTree params === // Ref + Parent = error (parent without URL) - { + "ostree-no-url": { false, "POST", "/api/v1/compose", @@ -1126,18 +1170,18 @@ func TestCompose(t *testing.T) { []string{"build_id", "warnings"}, }, // Valid Ref + URL = OK - { + "ostree-valid": { false, "POST", "/api/v1/compose", fmt.Sprintf(`{"blueprint_name": "test","compose_type":"%s","branch":"master","ostree":{"ref":"%s","parent":"","url":"%s"}}`, test_distro.TestImageTypeOSTree, ostreeRepoOther.OSTreeRef, ostreeRepoOther.Server.URL), http.StatusOK, `{"status": true}`, - expectedComposeOSTree, + expectedComposeOSTreeOther, []string{"build_id", "warnings"}, }, // Ref + invalid URL = error - { + "ostree-invalid-url": { false, "POST", "/api/v1/compose", @@ -1148,7 +1192,7 @@ func TestCompose(t *testing.T) { []string{"build_id", "warnings"}, }, // Bad Ref + URL = error - { + "ostree-bad-ref": { false, "POST", "/api/v1/compose", @@ -1159,7 +1203,7 @@ func TestCompose(t *testing.T) { []string{"build_id", "warnings"}, }, // Incorrect Ref + URL = the parameters are okay, but the ostree repo returns 404 - { + "ostree-404": { false, "POST", "/api/v1/compose", @@ -1170,18 +1214,18 @@ func TestCompose(t *testing.T) { []string{"build_id", "warnings"}, }, // Ref + Parent + URL = OK - { + "ostree-all-params": { false, "POST", "/api/v1/compose", fmt.Sprintf(`{"blueprint_name": "test","compose_type":"%s","branch":"master","ostree":{"ref":"%s","parent":"%s","url":"%s"}}`, test_distro.TestImageTypeOSTree, "the/new/ref", ostreeRepoOther.OSTreeRef, ostreeRepoOther.Server.URL), http.StatusOK, `{"status":true}`, - expectedComposeOSTree, + expectedComposeOSTreeOther, []string{"build_id", "warnings"}, }, // Parent + URL = OK - { + "ostree-parent-url": { false, "POST", "/api/v1/compose", @@ -1192,7 +1236,7 @@ func TestCompose(t *testing.T) { []string{"build_id", "warnings"}, }, // URL only = OK (uses default ref, so we need to specify URL for ostree repo with default ref) - { + "ostree-url-only": { false, "POST", "/api/v1/compose", @@ -1205,7 +1249,7 @@ func TestCompose(t *testing.T) { } tempdir := t.TempDir() - for _, c := range cases { + for name, c := range cases { api, s := createWeldrAPI(tempdir, rpmmd_mock.NoComposesFixture) test.TestRoute(t, api, c.External, c.Method, c.Path, c.Body, c.ExpectedStatus, c.ExpectedJSON, c.IgnoreFields...) @@ -1215,7 +1259,7 @@ func TestCompose(t *testing.T) { composes := s.GetAllComposes() - require.Equalf(t, 1, len(composes), "%s: bad compose count in store", c.Path) + require.Equalf(t, 1, len(composes), "%s: %s: bad compose count in store", name, c.Path) // I have no idea how to get the compose in better way var composeStruct store.Compose @@ -1224,10 +1268,10 @@ func TestCompose(t *testing.T) { break } - require.NotNilf(t, composeStruct.ImageBuild.Manifest, "%s: the compose in the store did not contain a blueprint", c.Path) + require.NotNilf(t, composeStruct.ImageBuild.Manifest, "%s: %s: the compose in the store did not contain a blueprint", name, c.Path) if diff := cmp.Diff(composeStruct, *c.ExpectedCompose, test.IgnoreDates(), test.IgnoreUuids(), test.Ignore("Targets.Options.Location"), test.CompareImageTypes()); diff != "" { - t.Errorf("%s: compose in store isn't the same as expected, diff:\n%s", c.Path, diff) + t.Errorf("%s: %s: compose in store isn't the same as expected, diff:\n%s", name, c.Path, diff) } } } @@ -1326,6 +1370,7 @@ func TestComposeLogs(t *testing.T) { t.Skip("This test is for internal testing only") } + emptyManifest := `{"version":"2","pipelines":[{"name":"build"},{"name":"os"}],"sources":{}}` var successCases = []struct { Path string ExpectedContentDisposition string @@ -1335,10 +1380,10 @@ func TestComposeLogs(t *testing.T) { }{ {"/api/v0/compose/logs/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002-logs.tar", "application/x-tar", "logs/osbuild.log", "The compose result is empty.\n"}, {"/api/v1/compose/logs/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002-logs.tar", "application/x-tar", "logs/osbuild.log", "The compose result is empty.\n"}, - {"/api/v0/compose/metadata/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002-metadata.tar", "application/x-tar", "30000000-0000-0000-0000-000000000002.json", `{"version":"2","pipelines":[],"sources":{}}`}, - {"/api/v1/compose/metadata/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002-metadata.tar", "application/x-tar", "30000000-0000-0000-0000-000000000002.json", `{"version":"2","pipelines":[],"sources":{}}`}, - {"/api/v0/compose/results/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002.tar", "application/x-tar", "30000000-0000-0000-0000-000000000002.json", `{"version":"2","pipelines":[],"sources":{}}`}, - {"/api/v1/compose/results/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002.tar", "application/x-tar", "30000000-0000-0000-0000-000000000002.json", `{"version":"2","pipelines":[],"sources":{}}`}, + {"/api/v0/compose/metadata/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002-metadata.tar", "application/x-tar", "30000000-0000-0000-0000-000000000002.json", emptyManifest}, + {"/api/v1/compose/metadata/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002-metadata.tar", "application/x-tar", "30000000-0000-0000-0000-000000000002.json", emptyManifest}, + {"/api/v0/compose/results/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002.tar", "application/x-tar", "30000000-0000-0000-0000-000000000002.json", emptyManifest}, + {"/api/v1/compose/results/30000000-0000-0000-0000-000000000002", "attachment; filename=30000000-0000-0000-0000-000000000002.tar", "application/x-tar", "30000000-0000-0000-0000-000000000002.json", emptyManifest}, } tempdir := t.TempDir() @@ -1998,7 +2043,8 @@ func TestComposePOST_ImageTypeDenylist(t *testing.T) { manifest, _, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, 0) require.NoError(t, err) - mf, err := manifest.Serialize(nil, nil, nil) + rPkgs, rContainers, rCommits := test_distro.ResolveContent(manifest.Content.PackageSets, manifest.Content.Containers, manifest.Content.OSTreeCommits) + mf, err := manifest.Serialize(rPkgs, rContainers, rCommits) require.NoError(t, err) expectedComposeLocal := &store.Compose{ diff --git a/internal/worker/server_test.go b/internal/worker/server_test.go index f9e91cab2..455426fe8 100644 --- a/internal/worker/server_test.go +++ b/internal/worker/server_test.go @@ -145,9 +145,10 @@ func TestCreate(t *testing.T) { _, err = server.EnqueueOSBuild(arch.Name(), &worker.OSBuildJob{Manifest: mf}, "") require.NoError(t, err) + emptyManifest := `{"version":"2","pipelines":[{"name":"build"},{"name":"os"}],"sources":{}}` test.TestRoute(t, handler, false, "POST", "/api/worker/v1/jobs", fmt.Sprintf(`{"types":["%s"],"arch":"%s"}`, worker.JobTypeOSBuild, test_distro.TestArchName), http.StatusCreated, - fmt.Sprintf(`{"kind":"RequestJob","href":"/api/worker/v1/jobs","type":"%s","args":{"manifest":{"version":"2","pipelines":[],"sources":{}}}}`, worker.JobTypeOSBuild), "id", "location", "artifact_location") + fmt.Sprintf(`{"kind":"RequestJob","href":"/api/worker/v1/jobs","type":"%s","args":{"manifest":`+emptyManifest+`}}`, worker.JobTypeOSBuild), "id", "location", "artifact_location") } func TestCancel(t *testing.T) {