From 4f51d447628b9a7024d70b9bc75b07cd1a617b29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Tue, 19 Sep 2023 18:00:23 +0200 Subject: [PATCH] Worker/koji-finalize: add cloud target results to image/build metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add any non-Koji upload target results attached to an OSBuild result, to the image extra metadata. This will make it easy to locate any image from Koji uploaded to cloud, in the target cloud environment. The rationale behind including only non-Koji target results is that one can find it only in Koji, so there is no added value in including the Koji target results at all. Extend the `koji.sh` to check the target results in image metadata when testing Koji scenario with cloud upload. Signed-off-by: Tomáš Hozza --- cmd/osbuild-worker/jobimpl-koji-finalize.go | 7 + internal/upload/koji/koji.go | 4 + internal/worker/json.go | 14 ++ internal/worker/json_test.go | 144 ++++++++++++++++++++ test/cases/koji.sh | 32 +++++ 5 files changed, 201 insertions(+) diff --git a/cmd/osbuild-worker/jobimpl-koji-finalize.go b/cmd/osbuild-worker/jobimpl-koji-finalize.go index d97a0eb41..5c3b4aac6 100644 --- a/cmd/osbuild-worker/jobimpl-koji-finalize.go +++ b/cmd/osbuild-worker/jobimpl-koji-finalize.go @@ -196,6 +196,13 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { imageFilename = args.KojiFilenames[i] } + // If there are any non-Koji target results in the build, + // add them to the image output extra metadata. + nonKojiTargetResults := buildResult.TargetResultsFilterByName([]target.TargetName{target.TargetNameKoji}) + if len(nonKojiTargetResults) > 0 { + imgOutputExtraInfo.UploadTargetResults = nonKojiTargetResults + } + imgOutputsExtraInfo[imageFilename] = imgOutputExtraInfo // Image output diff --git a/internal/upload/koji/koji.go b/internal/upload/koji/koji.go index e0b8356a3..eb1186e6c 100644 --- a/internal/upload/koji/koji.go +++ b/internal/upload/koji/koji.go @@ -27,6 +27,7 @@ import ( "github.com/ubccr/kerby/khttp" "github.com/osbuild/images/pkg/rpmmd" + "github.com/osbuild/osbuild-composer/internal/target" ) type Koji struct { @@ -120,6 +121,9 @@ type ImageExtraInfo struct { Arch string `json:"arch"` // Boot mode of the image BootMode string `json:"boot_mode,omitempty"` + // Results from any upload targets associated with the image + // except for the Koji target. + UploadTargetResults []*target.TargetResult `json:"upload_target_results,omitempty"` } func (ImageExtraInfo) isImageOutputTypeMD() {} diff --git a/internal/worker/json.go b/internal/worker/json.go index 8b5feddad..edd4c8c44 100644 --- a/internal/worker/json.go +++ b/internal/worker/json.go @@ -10,6 +10,7 @@ import ( "github.com/osbuild/images/pkg/rpmmd" "github.com/osbuild/osbuild-composer/internal/target" "github.com/osbuild/osbuild-composer/internal/worker/clienterrors" + "golang.org/x/exp/slices" ) // @@ -97,6 +98,19 @@ func (j *OSBuildJobResult) TargetResultsByName(name target.TargetName) []*target return targetResults } +// TargetResultsFilterByName iterates over TargetResults attached to the Job result and +// returns a slice of Target results excluding the provided names (types). If there were +// no TargetResults left after filtering, the returned slice will be empty. +func (j *OSBuildJobResult) TargetResultsFilterByName(excludeNames []target.TargetName) []*target.TargetResult { + targetResults := []*target.TargetResult{} + for _, targetResult := range j.TargetResults { + if !slices.Contains(excludeNames, targetResult.Name) { + targetResults = append(targetResults, targetResult) + } + } + return targetResults +} + func (j *FileResolveJobResult) ResolutionErrors() []*clienterrors.Error { resolutionErrors := []*clienterrors.Error{} diff --git a/internal/worker/json_test.go b/internal/worker/json_test.go index 310e1839f..b208516dd 100644 --- a/internal/worker/json_test.go +++ b/internal/worker/json_test.go @@ -189,6 +189,150 @@ func TestOSBuildJobResultTargetResultsByName(t *testing.T) { } } +func TestOSBuildJobResultTargetResultsFilterByName(t *testing.T) { + testCases := []struct { + jobResult OSBuildJobResult + targetNames []target.TargetName + targetResults []*target.TargetResult + }{ + { + jobResult: OSBuildJobResult{ + TargetResults: []*target.TargetResult{ + { + Name: target.TargetNameAWS, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "can't login to AWS", nil), + }, + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + { + Name: target.TargetNameAWSS3, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "failed to upload image to AWS S3", nil), + }, + }, + }, + targetNames: []target.TargetName{ + target.TargetNameVMWare, + }, + targetResults: []*target.TargetResult{ + { + Name: target.TargetNameAWS, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "can't login to AWS", nil), + }, + { + Name: target.TargetNameAWSS3, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "failed to upload image to AWS S3", nil), + }, + }, + }, + { + jobResult: OSBuildJobResult{ + TargetResults: []*target.TargetResult{ + { + Name: target.TargetNameAWS, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "can't login to AWS", nil), + }, + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + { + Name: target.TargetNameAWSS3, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "failed to upload image to AWS S3", nil), + }, + }, + }, + targetNames: []target.TargetName{ + target.TargetNameVMWare, + target.TargetNameAWSS3, + }, + targetResults: []*target.TargetResult{ + { + Name: target.TargetNameAWS, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "can't login to AWS", nil), + }, + }, + }, + { + jobResult: OSBuildJobResult{ + TargetResults: []*target.TargetResult{ + { + Name: target.TargetNameAWS, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "can't login to AWS", nil), + }, + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + { + Name: target.TargetNameAWSS3, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "failed to upload image to AWS S3", nil), + }, + }, + }, + targetNames: []target.TargetName{ + target.TargetNameAWS, + target.TargetNameAWSS3, + }, + targetResults: []*target.TargetResult{ + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + }, + }, + { + jobResult: OSBuildJobResult{ + TargetResults: []*target.TargetResult{ + { + Name: target.TargetNameAWS, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "can't login to AWS", nil), + }, + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + { + Name: target.TargetNameVMWare, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "can't upload image to VMWare", nil), + }, + { + Name: target.TargetNameAWSS3, + TargetError: clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, "failed to upload image to AWS S3", nil), + }, + }, + }, + targetNames: []target.TargetName{ + target.TargetNameAWS, + target.TargetNameVMWare, + target.TargetNameAWSS3, + }, + targetResults: []*target.TargetResult{}, + }, + } + + for _, testCase := range testCases { + assert.EqualValues(t, testCase.targetResults, testCase.jobResult.TargetResultsFilterByName(testCase.targetNames)) + } +} + func TestOSBuildJobExports(t *testing.T) { testCases := []struct { job *OSBuildJob diff --git a/test/cases/koji.sh b/test/cases/koji.sh index 00e8fb81b..3de331488 100755 --- a/test/cases/koji.sh +++ b/test/cases/koji.sh @@ -147,6 +147,38 @@ function verify_buildinfo() { echo "Unexpected number of images in the buildinfo. Want 1, got ${outputs_images_count}." exit 1 fi + + # Verify that the target results are present in the image output metadata + local target_results + target_results="$(echo "${outputs_images}" | jq -r '.[0].extra.image.upload_target_results')" + local target_results_count + target_results_count="$(echo "${target_results}" | jq 'length')" + if [ "$target_results_count" -ne 1 ]; then + echo "Unexpected number of target results in the buildinfo. Want 1, got ${target_results_count}." + exit 1 + fi + + local target_result_name + target_result_name="$(echo "${target_results}" | jq -r '.[0].name')" + local want_target_result_name + case ${target_cloud} in + "$CLOUD_PROVIDER_AWS") + want_target_result_name="org.osbuild.aws" + ;; + "$CLOUD_PROVIDER_GCP") + want_target_result_name="org.osbuild.gcp" + ;; + "$CLOUD_PROVIDER_AZURE") + want_target_result_name="org.osbuild.azure.image" + ;; + *) + echo "Unknown cloud provider: ${CLOUD_PROVIDER}" + exit 1 + esac + if [ "${target_result_name}" != "${want_target_result_name}" ]; then + echo "Unexpected target result in the buildinfo. Want '${want_target_result_name}', got '${target_result_name}'." + exit 1 + fi fi local outputs_manifests