From e0ec3a2a1c6d425ecdd42fb366742bcb44148fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Thu, 7 Sep 2023 10:25:58 +0200 Subject: [PATCH] Worker/koji-finalize: import osbuild manifest and log to Koji build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Import osbuild manifest and build log to the Koji build as outputs. Also note the respective filenames in the image output extra metadata. Note that the osbuild manifest is imported as a log file for now. Koji has very limited set of output types defined and I still need to determine the best way to use a custom output type in Koji instances (as other content generators do). Signed-off-by: Tomáš Hozza --- cmd/osbuild-koji-tests/main_test.go | 4 +-- cmd/osbuild-koji/main.go | 2 +- cmd/osbuild-worker/jobimpl-koji-finalize.go | 40 +++++++++++++++++++-- cmd/osbuild-worker/jobimpl-osbuild.go | 2 +- internal/upload/koji/koji.go | 33 ++++++++++++++--- test/cases/koji.sh | 23 +++++++++++- 6 files changed, 92 insertions(+), 12 deletions(-) diff --git a/cmd/osbuild-koji-tests/main_test.go b/cmd/osbuild-koji-tests/main_test.go index 0045037ad..16012b467 100644 --- a/cmd/osbuild-koji-tests/main_test.go +++ b/cmd/osbuild-koji-tests/main_test.go @@ -179,8 +179,8 @@ func TestKojiImport(t *testing.T) { Checksum: hash, Type: koji.BuildOutputTypeImage, RPMs: []rpmmd.RPM{}, - Extra: koji.BuildOutputExtra{ - Image: koji.ImageExtraInfo{ + Extra: &koji.BuildOutputExtra{ + ImageOutput: koji.ImageExtraInfo{ Arch: "noarch", BootMode: distro.BOOT_LEGACY.String(), }, diff --git a/cmd/osbuild-koji/main.go b/cmd/osbuild-koji/main.go index 72f19f2e8..26dbc84e7 100644 --- a/cmd/osbuild-koji/main.go +++ b/cmd/osbuild-koji/main.go @@ -99,7 +99,7 @@ func main() { Type: koji.BuildOutputTypeImage, RPMs: []rpmmd.RPM{}, Extra: &koji.BuildOutputExtra{ - Image: koji.ImageExtraInfo{ + ImageOutput: koji.ImageExtraInfo{ Arch: arch, BootMode: distro.BOOT_NONE.String(), // TODO: put the correct boot mode here }, diff --git a/cmd/osbuild-worker/jobimpl-koji-finalize.go b/cmd/osbuild-worker/jobimpl-koji-finalize.go index da6bdbd56..d97a0eb41 100644 --- a/cmd/osbuild-worker/jobimpl-koji-finalize.go +++ b/cmd/osbuild-worker/jobimpl-koji-finalize.go @@ -198,6 +198,7 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { imgOutputsExtraInfo[imageFilename] = imgOutputExtraInfo + // Image output outputs = append(outputs, koji.BuildOutput{ BuildRootID: uint64(i), Filename: imageFilename, @@ -208,9 +209,44 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { Type: koji.BuildOutputTypeImage, RPMs: imageRPMs, Extra: &koji.BuildOutputExtra{ - Image: imgOutputExtraInfo, + ImageOutput: imgOutputExtraInfo, }, }) + + // OSBuild manifest output + // TODO: Condition below is present for backward compatibility with old workers which don't upload the manifest. + // TODO: Remove the condition it in the future. + if kojiTargetOptions.OSBuildManifest != nil { + outputs = append(outputs, koji.BuildOutput{ + BuildRootID: uint64(i), + Filename: kojiTargetOptions.OSBuildManifest.Filename, + FileSize: kojiTargetOptions.OSBuildManifest.Size, + Arch: buildResult.Arch, + ChecksumType: koji.ChecksumType(kojiTargetOptions.OSBuildManifest.ChecksumType), + Checksum: kojiTargetOptions.OSBuildManifest.Checksum, + Type: koji.BuildOutputTypeManifest, + Extra: &koji.BuildOutputExtra{ + ImageOutput: koji.ManifestExtraInfo{ + Arch: buildResult.Arch, + }, + }, + }) + } + + // Build log output + // TODO: Condition below is present for backward compatibility with old workers which don't upload the log. + // TODO: Remove the condition it in the future. + if kojiTargetOptions.Log != nil { + outputs = append(outputs, koji.BuildOutput{ + BuildRootID: uint64(i), + Filename: kojiTargetOptions.Log.Filename, + FileSize: kojiTargetOptions.Log.Size, + Arch: "noarch", // log file is not architecture dependent + ChecksumType: koji.ChecksumType(kojiTargetOptions.Log.ChecksumType), + Checksum: kojiTargetOptions.Log.Checksum, + Type: koji.BuildOutputTypeLog, + }) + } } build := koji.Build{ @@ -222,7 +258,7 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { StartTime: int64(args.StartTime), EndTime: time.Now().Unix(), Extra: koji.BuildExtra{ - TypeInfo: koji.TypeInfo{ + TypeInfo: koji.TypeInfoBuild{ Image: imgOutputsExtraInfo, }, }, diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index bc8742aaa..b9067ee6e 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -908,7 +908,7 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { Checksum: imageHash, Size: imageSize, }, - OsbuildManifest: &target.KojiOutputInfo{ + OSBuildManifest: &target.KojiOutputInfo{ Filename: manifestFilename, ChecksumType: target.ChecksumTypeMD5, Checksum: manifestHash, diff --git a/internal/upload/koji/koji.go b/internal/upload/koji/koji.go index 5382ddb2f..e0b8356a3 100644 --- a/internal/upload/koji/koji.go +++ b/internal/upload/koji/koji.go @@ -37,10 +37,10 @@ type Koji struct { // BUILD METADATA -// TypeInfo is a map whose entries are the names of the build types +// TypeInfoBuild is a map whose entries are the names of the build types // used for the build, and the values are free-form maps containing // type-specific information for the build. -type TypeInfo struct { +type TypeInfoBuild struct { // Image holds extra metadata about all images built by the build. // It is a map whose keys are the filenames of the images, and // the values are the extra metadata for the image. @@ -51,7 +51,7 @@ type TypeInfo struct { // BuildExtra holds extra metadata associated with the build. // It is a free-form map, but must contain at least the 'typeinfo' key. type BuildExtra struct { - TypeInfo TypeInfo `json:"typeinfo"` + TypeInfo TypeInfoBuild `json:"typeinfo"` } // Build represents a Koji build and holds metadata about it. @@ -107,6 +107,10 @@ type BuildRoot struct { // OUTPUT METADATA +type ImageOutputTypeExtraInfo interface { + isImageOutputTypeMD() +} + // ImageExtraInfo holds extra metadata about the image. // This structure is shared for the Extra metadata of the output and the build. type ImageExtraInfo struct { @@ -118,16 +122,35 @@ type ImageExtraInfo struct { BootMode string `json:"boot_mode,omitempty"` } +func (ImageExtraInfo) isImageOutputTypeMD() {} + +// ManifestExtraInfo holds extra metadata about the osbuild manifest. +type ManifestExtraInfo struct { + // TODO: include osbuild-composer version which produced the manifest? + // TODO: include the vendored 'images' version? + + Arch string `json:"arch"` +} + +func (ManifestExtraInfo) isImageOutputTypeMD() {} + // BuildOutputExtra holds extra metadata associated with the build output. type BuildOutputExtra struct { - Image ImageExtraInfo `json:"image"` + // ImageOutput holds extra metadata about a single "image" output. + // "image" in this context is the "build type" in the Koji terminology, + // not necessarily an actual image. It can and must be used also for + // other supplementary files related to the image, such as osbuild manifest. + // The only exception are logs, which do not need to specify any "typeinfo". + ImageOutput ImageOutputTypeExtraInfo `json:"image"` } // BuildOutputType represents the type of a BuildOutput. type BuildOutputType string const ( - BuildOutputTypeImage BuildOutputType = "image" + BuildOutputTypeImage BuildOutputType = "image" + BuildOutputTypeLog BuildOutputType = "log" + BuildOutputTypeManifest BuildOutputType = "osbuild-manifest" ) // ChecksumType represents the type of a checksum used for a BuildOutput. diff --git a/test/cases/koji.sh b/test/cases/koji.sh index f9b33bdfc..00e8fb81b 100755 --- a/test/cases/koji.sh +++ b/test/cases/koji.sh @@ -132,7 +132,7 @@ function verify_buildinfo() { # extract the image archives paths from the output and keep only the filenames local outputs_images - outputs_images="$(koji -s "${KOJI_HUB_URL}" --noauth call --json listArchives "${buildid}" | jq -r 'map(select(.btype == "image"))')" + outputs_images="$(koji -s "${KOJI_HUB_URL}" --noauth call --json listArchives "${buildid}" | jq -r 'map(select(.btype == "image" and .type_name != "json"))')" # we build one image for cloud test case and two for non-cloud test case local outputs_images_count @@ -149,6 +149,24 @@ function verify_buildinfo() { fi fi + local outputs_manifests + outputs_manifests="$(koji -s "${KOJI_HUB_URL}" --noauth call --json listArchives "${buildid}" | jq -r 'map(select(.btype == "image" and .type_name == "json"))')" + local outputs_manifests_count + outputs_manifests_count="$(echo "${outputs_manifests}" | jq 'length')" + if [ "${outputs_manifests_count}" -ne "${outputs_images_count}" ]; then + echo "Mismatch between the number of image archives and image manifests in the buildinfo" + exit 1 + fi + + local outputs_logs + outputs_logs="$(koji -s "${KOJI_HUB_URL}" --noauth call --json getBuildLogs "${buildid}" | jq -r 'map(select(.name != "cg_import.log"))')" + local outputs_logs_count + outputs_logs_count="$(echo "${outputs_logs}" | jq 'length')" + if [ "${outputs_logs_count}" -ne "${outputs_images_count}" ]; then + echo "Mismatch between the number of image archives and image logs in the buildinfo" + exit 1 + fi + local build_extra_md_image build_extra_md_image="$(echo "${extra_build_metadata}" | jq -r '.typeinfo.image')" @@ -297,6 +315,9 @@ koji --server="${KOJI_HUB_URL}" --noauth call --json getBuild 1 greenprint "Show Koji build archives" koji --server="${KOJI_HUB_URL}" --noauth call --json listArchives 1 +greenprint "Show Koji build logs" +koji --server="${KOJI_HUB_URL}" --noauth call --json getBuildLogs 1 + greenprint "Verify the Koji build info and metadata" verify_buildinfo 1 "${CLOUD_PROVIDER}"