From af1c37340711d3ad34aa3d8ba6c5321a8bd716b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Fri, 4 Aug 2023 17:12:18 +0200 Subject: [PATCH] Koji: expose image output metadata in build extra metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose the extra metadata information for each image output stored in `ImageExtraInfo` also in the build extra metadata. The extra metadata for each image is nested under key corresponding to the image filename. Extend the Koji test case to check information in the buildinfo output and specifically check for all expected metadata in the build Extra metadata field. Signed-off-by: Tomáš Hozza --- cmd/osbuild-worker/jobimpl-koji-finalize.go | 36 +++++++----- internal/upload/koji/koji.go | 6 +- test/cases/koji.sh | 65 ++++++++++++++++++++- 3 files changed, 92 insertions(+), 15 deletions(-) diff --git a/cmd/osbuild-worker/jobimpl-koji-finalize.go b/cmd/osbuild-worker/jobimpl-koji-finalize.go index 359600f2c..a2b503faf 100644 --- a/cmd/osbuild-worker/jobimpl-koji-finalize.go +++ b/cmd/osbuild-worker/jobimpl-koji-finalize.go @@ -115,17 +115,10 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { return err } - build := koji.Build{ - TaskID: args.TaskID, - Name: args.Name, - Version: args.Version, - Release: args.Release, - StartTime: int64(args.StartTime), - EndTime: time.Now().Unix(), - } - var buildRoots []koji.BuildRoot var outputs []koji.BuildOutput + // Extra info for each image output is stored using the image filename as the key + imgOutputsExtraInfo := map[string]koji.ImageExtraInfo{} var osbuildResults []worker.OSBuildJobResult initArgs, osbuildResults, err = extractDynamicArgs(job) @@ -133,7 +126,6 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { kojiFinalizeJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorParsingDynamicArgs, "Error parsing dynamic args", err.Error()) return err } - build.BuildID = initArgs.BuildID // Check the dependencies early. if hasFailedDependency(*initArgs, osbuildResults) { @@ -191,6 +183,11 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { // deduplicate imageRPMs = rpmmd.DeduplicateRPMs(imageRPMs) + imgOutputExtraInfo := koji.ImageExtraInfo{ + Arch: buildArgs.Arch, + } + imgOutputsExtraInfo[args.KojiFilenames[i]] = imgOutputExtraInfo + outputs = append(outputs, koji.BuildOutput{ BuildRootID: uint64(i), Filename: args.KojiFilenames[i], @@ -201,13 +198,26 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { Type: koji.BuildOutputTypeImage, RPMs: imageRPMs, Extra: koji.BuildOutputExtra{ - Image: koji.ImageExtraInfo{ - Arch: buildArgs.Arch, - }, + Image: imgOutputExtraInfo, }, }) } + build := koji.Build{ + BuildID: initArgs.BuildID, + TaskID: args.TaskID, + Name: args.Name, + Version: args.Version, + Release: args.Release, + StartTime: int64(args.StartTime), + EndTime: time.Now().Unix(), + Extra: koji.BuildExtra{ + TypeInfo: koji.TypeInfo{ + Image: imgOutputsExtraInfo, + }, + }, + } + err = impl.kojiImport(args.Server, build, buildRoots, outputs, args.KojiDirectory, initArgs.Token) if err != nil { kojiFinalizeJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorKojiFinalize, err.Error(), nil) diff --git a/internal/upload/koji/koji.go b/internal/upload/koji/koji.go index 1659accf7..d4a0b6e3a 100644 --- a/internal/upload/koji/koji.go +++ b/internal/upload/koji/koji.go @@ -41,7 +41,11 @@ type Koji struct { // used for the build, and the values are free-form maps containing // type-specific information for the build. type TypeInfo struct { - Image struct{} `json:"image"` + // Image holds extra metadata about all images built by the build. + // It is a map whose keys are the filenames of the images, and + // the values are the extra metadata for the image. + // There can't be more than one image with the same filename. + Image map[string]ImageExtraInfo `json:"image"` } // BuildExtra holds extra metadata associated with the build. diff --git a/test/cases/koji.sh b/test/cases/koji.sh index 8897b90de..c16b89eb7 100755 --- a/test/cases/koji.sh +++ b/test/cases/koji.sh @@ -21,6 +21,9 @@ CLOUD_PROVIDER_AWS="aws" CLOUD_PROVIDER_GCP="gcp" CLOUD_PROVIDER_AZURE="azure" +# define the default cloud provider to test for +CLOUD_PROVIDER="none" + # # Test types # @@ -109,6 +112,60 @@ fi trap cleanups EXIT +# Verify that all the expected information is present in the buildinfo +function verify_buildinfo() { + local buildinfo="${1}" + local target_cloud="${2:-none}" + + local extra_build_metadata + # extract the extra build metadata JSON from the output + extra_build_metadata="$(echo "${buildinfo}" | grep -oP '(?<=Extra: ).*' | tr "'" '"')" + + # sanity check the extra build metadata + if [ -z "${extra_build_metadata}" ]; then + echo "Extra build metadata is empty" + exit 1 + fi + + # extract the image archives paths from the output and keep only the filenames + local outputs_images + outputs_images="$(echo "${buildinfo}" | + sed -zE 's/.*Image archives:\n((\S+\n){1,})([\w\s]+:){0,}.*/\1/g' | + sed -E 's/.*\/(.*)/\1/g')" + + # we build one image for cloud test case and two for non-cloud test case + if [ "${target_cloud}" == "none" ]; then + if [[ $(echo "${outputs_images}" | wc -l) -ne 2 ]]; then + echo "Unexpected number of images in the buildinfo" + exit 1 + fi + else + if [[ $(echo "${outputs_images}" | wc -l) -ne 1 ]]; then + echo "Unexpected number of images in the buildinfo" + exit 1 + fi + fi + + local images_metadata + images_metadata="$(echo "${extra_build_metadata}" | jq -r '.typeinfo.image')" + + for image in $outputs_images; do + local image_metadata + image_metadata="$(echo "${images_metadata}" | jq -r ".\"${image}\"")" + if [ "${image_metadata}" == "null" ]; then + echo "Image metadata for '${image}' is missing" + exit 1 + fi + + local image_arch + image_arch="$(echo "${image_metadata}" | jq -r '.arch')" + if [ "${image_arch}" != "${ARCH}" ]; then + echo "Unexpected arch for '${image}'. Expected '${ARCH}', but got '${image_arch}'" + exit 1 + fi + done +} + # Provision the software under test. /usr/libexec/osbuild-composer-test/provision.sh jwt @@ -201,7 +258,13 @@ fi greenprint "Show Koji task" koji --server=http://localhost:8080/kojihub taskinfo 1 -koji --server=http://localhost:8080/kojihub buildinfo 1 + +greenprint "Show Koji buildinfo" +BUILDINFO_OUTPUT="$(koji --server=http://localhost:8080/kojihub buildinfo 1)" +echo "${BUILDINFO_OUTPUT}" + +greenprint "Verify the buildinfo output" +verify_buildinfo "${BUILDINFO_OUTPUT}" "${CLOUD_PROVIDER}" greenprint "Run the integration test" sudo /usr/libexec/osbuild-composer-test/osbuild-koji-tests