From c5a494613512df710665c464caafcf6597d44b66 Mon Sep 17 00:00:00 2001 From: Tomas Hozza Date: Thu, 18 Nov 2021 15:07:25 +0100 Subject: [PATCH] Test all manifests with depsolved package sets Generated image test case manifests for all supported distros, arches and image-types are being tested as part of distro unit tests. However due to time constrains, the unit test does not depsolve the image's default package sets and thus does not check if they changed in the internal osbuild-composer's representation, compared to the generated image test case. Extend the `TestDistro_Manifest()` function used by the unit test to allow depsolving image's package sets. Introduce a new test case binary `osbuild-composer-manifest-tests` allowing to check the manifests generated by composer for all supported combinations of images against generated manifests, including depsolving image's default package sets. Introduce a new CI test case `manifest_tests.sh` executing the `osbuild-composer-manifest-tests` binary and testing all existing image test cases. Run it in CI on RHEL-9 runner. Modify SPEC file to ship the newly added test case. Signed-off-by: Tomas Hozza --- .gitlab-ci.yml | 21 +++++++++ Makefile | 1 + .../main_test.go | 39 ++++++++++++++++ internal/distro/distro_test.go | 3 ++ .../distro_test_common/distro_test_common.go | 45 ++++++++++++++++--- osbuild-composer.spec | 2 + test/cases/manifest_tests.sh | 36 +++++++++++++++ 7 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 cmd/osbuild-composer-manifest-tests/main_test.go create mode 100755 test/cases/manifest_tests.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ebaec282..e3258315c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -137,6 +137,27 @@ Base: - "*.repo" when: always +Manifests: + stage: test + extends: .terraform + rules: + - if: '$CI_PIPELINE_SOURCE != "schedule"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-9.0-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "9"' + - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.6-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "8"' + script: + - schutzbot/deploy.sh + - /usr/libexec/tests/osbuild-composer/manifest_tests.sh + parallel: + matrix: + - RUNNER: + - aws/rhel-9.0-nightly-x86_64 + INTERNAL_NETWORK: ["true"] + artifacts: + paths: + - journal-log.gpg + - "*.repo" + when: always + Regression: stage: test extends: .terraform diff --git a/Makefile b/Makefile index f15810e04..6c8d23bf2 100644 --- a/Makefile +++ b/Makefile @@ -123,6 +123,7 @@ build: go test -c -tags=integration -o bin/osbuild-auth-tests ./cmd/osbuild-auth-tests/ go test -c -tags=integration -o bin/osbuild-koji-tests ./cmd/osbuild-koji-tests/ go test -c -tags=integration -o bin/osbuild-composer-dbjobqueue-tests ./cmd/osbuild-composer-dbjobqueue-tests/ + go test -c -tags=integration -o bin/osbuild-composer-manifest-tests ./cmd/osbuild-composer-manifest-tests/ .PHONY: install install: diff --git a/cmd/osbuild-composer-manifest-tests/main_test.go b/cmd/osbuild-composer-manifest-tests/main_test.go new file mode 100644 index 000000000..850ba1689 --- /dev/null +++ b/cmd/osbuild-composer-manifest-tests/main_test.go @@ -0,0 +1,39 @@ +// +build integration + +package main + +import ( + "flag" + "os" + "testing" + + "github.com/osbuild/osbuild-composer/internal/distro/distro_test_common" + "github.com/osbuild/osbuild-composer/internal/distroregistry" + "github.com/stretchr/testify/require" +) + +var manifestsPath string +var dnfJsonPath string +var testCaseGlob string + +func init() { + flag.StringVar(&manifestsPath, "manifests-path", "/usr/share/tests/osbuild-composer/manifests", "path to a directory with *.json files containing image test cases") + flag.StringVar(&dnfJsonPath, "dnf-json-path", "/usr/libexec/osbuild-composer/dnf-json", "path to the 'dnf-json' executable") + flag.StringVar(&testCaseGlob, "test-case-glob", "*", "glob pattern to select image test cases to verify") +} + +func TestManifests(t *testing.T) { + cacheDirPath := "/var/tmp/osbuild-composer-manifest-tests/rpmmd" + err := os.MkdirAll(cacheDirPath, 0755) + require.Nilf(t, err, "failed to create RPMMD cache directory %q", cacheDirPath) + + distro_test_common.TestDistro_Manifest( + t, + manifestsPath, + testCaseGlob, + distroregistry.NewDefault(), + true, + cacheDirPath, + dnfJsonPath, + ) +} diff --git a/internal/distro/distro_test.go b/internal/distro/distro_test.go index f2dcdcb35..be547ad58 100644 --- a/internal/distro/distro_test.go +++ b/internal/distro/distro_test.go @@ -16,6 +16,9 @@ func TestDistro_Manifest(t *testing.T) { "../../test/data/manifests/", "*", distroregistry.NewDefault(), + false, // This test case does not check for changes in the imageType package sets! + "", + "", ) } diff --git a/internal/distro/distro_test_common/distro_test_common.go b/internal/distro/distro_test_common/distro_test_common.go index 295cf7947..6a1437be4 100644 --- a/internal/distro/distro_test_common/distro_test_common.go +++ b/internal/distro/distro_test_common/distro_test_common.go @@ -20,7 +20,7 @@ import ( const RandomTestSeed = 0 -func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, registry *distroregistry.Registry) { +func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, registry *distroregistry.Registry, depsolvePkgSets bool, dnfCacheDir, dnfJsonPath string) { assert := assert.New(t) fileNames, err := filepath.Glob(filepath.Join(pipelinePath, prefix)) assert.NoErrorf(err, "Could not read pipelines directory '%s': %v", pipelinePath, err) @@ -41,9 +41,9 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis Blueprint *blueprint.Blueprint `json:"blueprint"` } var tt struct { - ComposeRequest *composeRequest `json:"compose-request"` - PackageSets map[string][]rpmmd.PackageSpec `json:"rpmmd"` - Manifest distro.Manifest `json:"manifest,omitempty"` + ComposeRequest *composeRequest `json:"compose-request"` + PackageSpecSets map[string][]rpmmd.PackageSpec `json:"rpmmd"` + Manifest distro.Manifest `json:"manifest,omitempty"` } file, err := ioutil.ReadFile(fileName) assert.NoErrorf(err, "Could not read test-case '%s': %v", fileName, err) @@ -82,6 +82,24 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis t.Errorf("unknown image type: %v", tt.ComposeRequest.ImageType) return } + + var imgPackageSpecSets map[string][]rpmmd.PackageSpec + // depsolve the image's package set to catch changes in the image's default package set. + // downside is that this takes long time + if depsolvePkgSets { + require.NotEmptyf(t, dnfCacheDir, "DNF cache directory path must be provided when chosen to depsolve image package sets") + require.NotEmptyf(t, dnfJsonPath, "path to 'dnf-json' must be provided when chosen to depsolve image package sets") + imgPackageSpecSets = getImageTypePkgSpecSets( + imageType, + *tt.ComposeRequest.Blueprint, + repos, + dnfCacheDir, + dnfJsonPath, + ) + } else { + imgPackageSpecSets = tt.PackageSpecSets + } + got, err := imageType.Manifest(tt.ComposeRequest.Blueprint.Customizations, distro.ImageOptions{ Size: imageType.Size(0), @@ -90,7 +108,7 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis }, }, repos, - tt.PackageSets, + imgPackageSpecSets, RandomTestSeed) if (err == nil && tt.Manifest == nil) || (err != nil && tt.Manifest != nil) { @@ -111,6 +129,23 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis } } +func getImageTypePkgSpecSets(imageType distro.ImageType, bp blueprint.Blueprint, repos []rpmmd.RepoConfig, cacheDir, dnfJsonPath string) map[string][]rpmmd.PackageSpec { + imgPackageSets := imageType.PackageSets(bp) + + rpm_md := rpmmd.NewRPMMD(cacheDir, dnfJsonPath) + + imgPackageSpecSets := make(map[string][]rpmmd.PackageSpec) + for name, packages := range imgPackageSets { + packageSpecs, _, err := rpm_md.Depsolve(packages, repos, imageType.Arch().Distro().ModulePlatformID(), imageType.Arch().Name(), imageType.Arch().Distro().Releasever()) + if err != nil { + panic("Could not depsolve: " + err.Error()) + } + imgPackageSpecSets[name] = packageSpecs + } + + return imgPackageSpecSets +} + func isOSTree(imgType distro.ImageType) bool { packageSets := imgType.PackageSets(blueprint.Blueprint{}) for _, pkg := range packageSets["build-packages"].Include { diff --git a/osbuild-composer.spec b/osbuild-composer.spec index 87dde25aa..972dc6cf5 100644 --- a/osbuild-composer.spec +++ b/osbuild-composer.spec @@ -159,6 +159,7 @@ go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-image-te go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-auth-tests %{goipath}/cmd/osbuild-auth-tests go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-koji-tests %{goipath}/cmd/osbuild-koji-tests go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-composer-dbjobqueue-tests %{goipath}/cmd/osbuild-composer-dbjobqueue-tests +go test -c -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-composer-manifest-tests %{goipath}/cmd/osbuild-composer-manifest-tests go build -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/cloud-cleaner %{goipath}/cmd/cloud-cleaner go build -tags=integration -ldflags="${TEST_LDFLAGS}" -o _bin/osbuild-mock-openid-provider %{goipath}/cmd/osbuild-mock-openid-provider @@ -224,6 +225,7 @@ install -m 0755 -vp _bin/osbuild-image-tests %{buildroot}%{_l install -m 0755 -vp _bin/osbuild-auth-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp _bin/osbuild-koji-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp _bin/osbuild-composer-dbjobqueue-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/ +install -m 0755 -vp _bin/osbuild-composer-manifest-tests %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp _bin/cloud-cleaner %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp _bin/osbuild-mock-openid-provider %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/define-compose-url.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/ diff --git a/test/cases/manifest_tests.sh b/test/cases/manifest_tests.sh new file mode 100755 index 000000000..da348af7b --- /dev/null +++ b/test/cases/manifest_tests.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -euo pipefail + +MANIFEST_TESTS_RUNNER="/usr/libexec/osbuild-composer-test/osbuild-composer-manifest-tests" +DNF_JSON_PATH="/usr/libexec/osbuild-composer/dnf-json" +IMAGE_TEST_CASES_PATH="/usr/share/tests/osbuild-composer/manifests" + +WORKING_DIRECTORY=/usr/libexec/osbuild-composer +mkdir --parents /tmp/logs +LOGS_DIRECTORY=$(mktemp --directory --tmpdir=/tmp/logs) + +# Print out a nice test divider so we know when tests stop and start. +test_divider () { + printf "%0.s-" {1..78} && echo +} + +# Provision the software under test. +/usr/libexec/osbuild-composer-test/provision.sh + +# Change to the working directory. +cd $WORKING_DIRECTORY + +# Run test case. +TEST_NAME=$(basename "$MANIFEST_TESTS_RUNNER") +echo +test_divider +echo "🏃🏻 Running test: ${TEST_NAME}" +test_divider + +if sudo "$MANIFEST_TESTS_RUNNER" -test.v -manifests-path "$IMAGE_TEST_CASES_PATH" -dnf-json-path "$DNF_JSON_PATH" | tee "${LOGS_DIRECTORY}"/"${TEST_NAME}".log; then + echo "🎉 Test passed." + exit 0 +else + echo "🔥 Test failed." + exit 1 +fi