Test all manifests with depsolved package sets

Generated image test case manifests for all supported distros, arches and
image-types are being tested as part of distro unit tests. However due
to time constrains, the unit test does not depsolve the image's default
package sets and thus does not check if they changed in the internal
osbuild-composer's representation, compared to the generated image test
case.

Extend the `TestDistro_Manifest()` function used by the unit test to
allow depsolving image's package sets.

Introduce a new test case binary `osbuild-composer-manifest-tests`
allowing to check the manifests generated by composer for all supported
combinations of images against generated manifests, including depsolving
image's default package sets.

Introduce a new CI test case `manifest_tests.sh` executing the
`osbuild-composer-manifest-tests` binary and testing all existing image
test cases. Run it in CI on RHEL-9 runner.

Modify SPEC file to ship the newly added test case.

Signed-off-by: Tomas Hozza <thozza@redhat.com>

.gitlab-ci.yml

@@ -137,6 +137,27 @@ Base:
       - "*.repo"
     when: always
 
+Manifests:
+  stage: test
+  extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-9.0-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "9"'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.6-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "8"'
+  script:
+    - schutzbot/deploy.sh
+    - /usr/libexec/tests/osbuild-composer/manifest_tests.sh
+  parallel:
+    matrix:
+      - RUNNER:
+          - aws/rhel-9.0-nightly-x86_64
+        INTERNAL_NETWORK: ["true"]
+  artifacts:
+    paths:
+      - journal-log.gpg
+      - "*.repo"
+    when: always
+
 Regression:
   stage: test
   extends: .terraform

Makefile

@@ -123,6 +123,7 @@ build:
 	go test -c -tags=integration -o bin/osbuild-auth-tests ./cmd/osbuild-auth-tests/
 	go test -c -tags=integration -o bin/osbuild-koji-tests ./cmd/osbuild-koji-tests/
 	go test -c -tags=integration -o bin/osbuild-composer-dbjobqueue-tests ./cmd/osbuild-composer-dbjobqueue-tests/
+	go test -c -tags=integration -o bin/osbuild-composer-manifest-tests ./cmd/osbuild-composer-manifest-tests/
 
 .PHONY: install
 install:

cmd/osbuild-composer-manifest-tests/main_test.go

@@ -0,0 +1,39 @@
+// +build integration
+
+package main
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/osbuild/osbuild-composer/internal/distro/distro_test_common"
+	"github.com/osbuild/osbuild-composer/internal/distroregistry"
+	"github.com/stretchr/testify/require"
+)
+
+var manifestsPath string
+var dnfJsonPath string
+var testCaseGlob string
+
+func init() {
+	flag.StringVar(&manifestsPath, "manifests-path", "/usr/share/tests/osbuild-composer/manifests", "path to a directory with *.json files containing image test cases")
+	flag.StringVar(&dnfJsonPath, "dnf-json-path", "/usr/libexec/osbuild-composer/dnf-json", "path to the 'dnf-json' executable")
+	flag.StringVar(&testCaseGlob, "test-case-glob", "*", "glob pattern to select image test cases to verify")
+}
+
+func TestManifests(t *testing.T) {
+	cacheDirPath := "/var/tmp/osbuild-composer-manifest-tests/rpmmd"
+	err := os.MkdirAll(cacheDirPath, 0755)
+	require.Nilf(t, err, "failed to create RPMMD cache directory %q", cacheDirPath)
+
+	distro_test_common.TestDistro_Manifest(
+		t,
+		manifestsPath,
+		testCaseGlob,
+		distroregistry.NewDefault(),
+		true,
+		cacheDirPath,
+		dnfJsonPath,
+	)
+}

internal/distro/distro_test.go

@@ -16,6 +16,9 @@ func TestDistro_Manifest(t *testing.T) {
 		"../../test/data/manifests/",
 		"*",
 		distroregistry.NewDefault(),
+		false, // This test case does not check for changes in the imageType package sets!
+		"",
+		"",
 	)
 }
 

internal/distro/distro_test_common/distro_test_common.go

@@ -20,7 +20,7 @@ import (
 
 const RandomTestSeed = 0
 
-func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, registry *distroregistry.Registry) {
+func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, registry *distroregistry.Registry, depsolvePkgSets bool, dnfCacheDir, dnfJsonPath string) {
 	assert := assert.New(t)
 	fileNames, err := filepath.Glob(filepath.Join(pipelinePath, prefix))
 	assert.NoErrorf(err, "Could not read pipelines directory '%s': %v", pipelinePath, err)
@@ -41,9 +41,9 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis
 			Blueprint    *blueprint.Blueprint `json:"blueprint"`
 		}
 		var tt struct {
-			ComposeRequest *composeRequest                `json:"compose-request"`
-			PackageSets    map[string][]rpmmd.PackageSpec `json:"rpmmd"`
-			Manifest       distro.Manifest                `json:"manifest,omitempty"`
+			ComposeRequest  *composeRequest                `json:"compose-request"`
+			PackageSpecSets map[string][]rpmmd.PackageSpec `json:"rpmmd"`
+			Manifest        distro.Manifest                `json:"manifest,omitempty"`
 		}
 		file, err := ioutil.ReadFile(fileName)
 		assert.NoErrorf(err, "Could not read test-case '%s': %v", fileName, err)
@@ -82,6 +82,24 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis
 				t.Errorf("unknown image type: %v", tt.ComposeRequest.ImageType)
 				return
 			}
+
+			var imgPackageSpecSets map[string][]rpmmd.PackageSpec
+			// depsolve the image's package set to catch changes in the image's default package set.
+			// downside is that this takes long time
+			if depsolvePkgSets {
+				require.NotEmptyf(t, dnfCacheDir, "DNF cache directory path must be provided when chosen to depsolve image package sets")
+				require.NotEmptyf(t, dnfJsonPath, "path to 'dnf-json' must be provided when chosen to depsolve image package sets")
+				imgPackageSpecSets = getImageTypePkgSpecSets(
+					imageType,
+					*tt.ComposeRequest.Blueprint,
+					repos,
+					dnfCacheDir,
+					dnfJsonPath,
+				)
+			} else {
+				imgPackageSpecSets = tt.PackageSpecSets
+			}
+
 			got, err := imageType.Manifest(tt.ComposeRequest.Blueprint.Customizations,
 				distro.ImageOptions{
 					Size: imageType.Size(0),
@@ -90,7 +108,7 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis
 					},
 				},
 				repos,
-				tt.PackageSets,
+				imgPackageSpecSets,
 				RandomTestSeed)
 
 			if (err == nil && tt.Manifest == nil) || (err != nil && tt.Manifest != nil) {
@@ -111,6 +129,23 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis
 	}
 }
 
+func getImageTypePkgSpecSets(imageType distro.ImageType, bp blueprint.Blueprint, repos []rpmmd.RepoConfig, cacheDir, dnfJsonPath string) map[string][]rpmmd.PackageSpec {
+	imgPackageSets := imageType.PackageSets(bp)
+
+	rpm_md := rpmmd.NewRPMMD(cacheDir, dnfJsonPath)
+
+	imgPackageSpecSets := make(map[string][]rpmmd.PackageSpec)
+	for name, packages := range imgPackageSets {
+		packageSpecs, _, err := rpm_md.Depsolve(packages, repos, imageType.Arch().Distro().ModulePlatformID(), imageType.Arch().Name(), imageType.Arch().Distro().Releasever())
+		if err != nil {
+			panic("Could not depsolve: " + err.Error())
+		}
+		imgPackageSpecSets[name] = packageSpecs
+	}
+
+	return imgPackageSpecSets
+}
+
 func isOSTree(imgType distro.ImageType) bool {
 	packageSets := imgType.PackageSets(blueprint.Blueprint{})
 	for _, pkg := range packageSets["build-packages"].Include {

osbuild-composer.spec

@@ -159,6 +159,7 @@ go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-image-te
 go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-auth-tests %{goipath}/cmd/osbuild-auth-tests
 go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-koji-tests %{goipath}/cmd/osbuild-koji-tests
 go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-composer-dbjobqueue-tests %{goipath}/cmd/osbuild-composer-dbjobqueue-tests
+go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-composer-manifest-tests %{goipath}/cmd/osbuild-composer-manifest-tests
 go build -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/cloud-cleaner %{goipath}/cmd/cloud-cleaner
 go build -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-mock-openid-provider %{goipath}/cmd/osbuild-mock-openid-provider
 
@@ -224,6 +225,7 @@ install -m 0755 -vp _bin/osbuild-image-tests                    %{buildroot}%{_l
 install -m 0755 -vp _bin/osbuild-auth-tests                     %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp _bin/osbuild-koji-tests                     %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp _bin/osbuild-composer-dbjobqueue-tests      %{buildroot}%{_libexecdir}/osbuild-composer-test/
+install -m 0755 -vp _bin/osbuild-composer-manifest-tests        %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp _bin/cloud-cleaner                          %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp _bin/osbuild-mock-openid-provider           %{buildroot}%{_libexecdir}/osbuild-composer-test/
 install -m 0755 -vp tools/define-compose-url.sh                 %{buildroot}%{_libexecdir}/osbuild-composer-test/

test/cases/manifest_tests.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -euo pipefail
+
+MANIFEST_TESTS_RUNNER="/usr/libexec/osbuild-composer-test/osbuild-composer-manifest-tests"
+DNF_JSON_PATH="/usr/libexec/osbuild-composer/dnf-json"
+IMAGE_TEST_CASES_PATH="/usr/share/tests/osbuild-composer/manifests"
+
+WORKING_DIRECTORY=/usr/libexec/osbuild-composer
+mkdir --parents /tmp/logs
+LOGS_DIRECTORY=$(mktemp --directory --tmpdir=/tmp/logs)
+
+# Print out a nice test divider so we know when tests stop and start.
+test_divider () {
+    printf "%0.s-" {1..78} && echo
+}
+
+# Provision the software under test.
+/usr/libexec/osbuild-composer-test/provision.sh
+
+# Change to the working directory.
+cd $WORKING_DIRECTORY
+
+# Run test case.
+TEST_NAME=$(basename "$MANIFEST_TESTS_RUNNER")
+echo
+test_divider
+echo "🏃🏻 Running test: ${TEST_NAME}"
+test_divider
+
+if sudo "$MANIFEST_TESTS_RUNNER" -test.v -manifests-path "$IMAGE_TEST_CASES_PATH" -dnf-json-path "$DNF_JSON_PATH" | tee "${LOGS_DIRECTORY}"/"${TEST_NAME}".log; then
+    echo "🎉  Test passed."
+    exit 0
+else
+    echo "🔥 Test failed."
+    exit 1
+fi