From 350762497f2fd09f7ac203e7230366d0db3bb224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Tue, 1 Aug 2023 17:23:33 +0200 Subject: [PATCH] Koji: refactor and consolidate structures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original Koji implementation expected that the output of a content generator is only an image. While in reality, we will eventually upload other types of files as outputs to Koji, such as logs and osbuild manifest. Rename Koji structures and their members to better map to the upstream Koji documentation and their JSON representation. Add comments to structures. Define type aliases and constants for string values which are more like enums, than a free-form values. These changes have no effect on the actual JSON representation of any of the structures Signed-off-by: Tomáš Hozza --- cmd/osbuild-koji-tests/main_test.go | 14 +-- cmd/osbuild-koji/main.go | 14 +-- cmd/osbuild-worker/jobimpl-koji-finalize.go | 24 ++--- internal/upload/koji/koji.go | 109 ++++++++++++++------ 4 files changed, 105 insertions(+), 56 deletions(-) diff --git a/cmd/osbuild-koji-tests/main_test.go b/cmd/osbuild-koji-tests/main_test.go index b11a82e54..ed7120b0a 100644 --- a/cmd/osbuild-koji-tests/main_test.go +++ b/cmd/osbuild-koji-tests/main_test.go @@ -141,7 +141,7 @@ func TestKojiImport(t *testing.T) { require.NoError(t, err) // Import the build - build := koji.ImageBuild{ + build := koji.Build{ TaskID: 1, Name: buildName, Version: "1", @@ -168,18 +168,18 @@ func TestKojiImport(t *testing.T) { RPMs: []rpmmd.RPM{}, }, } - output := []koji.Image{ + output := []koji.BuildOutput{ { BuildRootID: 1, Filename: filename, FileSize: uint64(filesize), Arch: "noarch", - ChecksumType: "md5", - MD5: hash, - Type: "image", + ChecksumType: koji.ChecksumTypeMD5, + Checksum: hash, + Type: koji.BuildOutputTypeImage, RPMs: []rpmmd.RPM{}, - Extra: koji.ImageExtra{ - Info: koji.ImageExtraInfo{ + Extra: koji.BuildOutputExtra{ + Image: koji.ImageExtraInfo{ Arch: "noarch", }, }, diff --git a/cmd/osbuild-koji/main.go b/cmd/osbuild-koji/main.go index e22dfc3fb..d84ecfb9e 100644 --- a/cmd/osbuild-koji/main.go +++ b/cmd/osbuild-koji/main.go @@ -60,7 +60,7 @@ func main() { return } - build := koji.ImageBuild{ + build := koji.Build{ TaskID: uint64(taskID), Name: name, Version: version, @@ -87,18 +87,18 @@ func main() { RPMs: []rpmmd.RPM{}, }, } - output := []koji.Image{ + output := []koji.BuildOutput{ { BuildRootID: 1, Filename: path.Base(filename), FileSize: length, Arch: arch, - ChecksumType: "md5", - MD5: hash, - Type: "image", + ChecksumType: koji.ChecksumTypeMD5, + Checksum: hash, + Type: koji.BuildOutputTypeImage, RPMs: []rpmmd.RPM{}, - Extra: koji.ImageExtra{ - Info: koji.ImageExtraInfo{ + Extra: koji.BuildOutputExtra{ + Image: koji.ImageExtraInfo{ Arch: arch, }, }, diff --git a/cmd/osbuild-worker/jobimpl-koji-finalize.go b/cmd/osbuild-worker/jobimpl-koji-finalize.go index c7e8d2c7a..359600f2c 100644 --- a/cmd/osbuild-worker/jobimpl-koji-finalize.go +++ b/cmd/osbuild-worker/jobimpl-koji-finalize.go @@ -21,9 +21,9 @@ type KojiFinalizeJobImpl struct { func (impl *KojiFinalizeJobImpl) kojiImport( server string, - build koji.ImageBuild, + build koji.Build, buildRoots []koji.BuildRoot, - images []koji.Image, + outputs []koji.BuildOutput, directory, token string) error { serverURL, err := url.Parse(server) @@ -48,7 +48,7 @@ func (impl *KojiFinalizeJobImpl) kojiImport( } }() - _, err = k.CGImport(build, buildRoots, images, directory, token) + _, err = k.CGImport(build, buildRoots, outputs, directory, token) if err != nil { return fmt.Errorf("Could not import build into koji: %v", err) } @@ -115,7 +115,7 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { return err } - build := koji.ImageBuild{ + build := koji.Build{ TaskID: args.TaskID, Name: args.Name, Version: args.Version, @@ -125,7 +125,7 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { } var buildRoots []koji.BuildRoot - var images []koji.Image + var outputs []koji.BuildOutput var osbuildResults []worker.OSBuildJobResult initArgs, osbuildResults, err = extractDynamicArgs(job) @@ -191,24 +191,24 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error { // deduplicate imageRPMs = rpmmd.DeduplicateRPMs(imageRPMs) - images = append(images, koji.Image{ + outputs = append(outputs, koji.BuildOutput{ BuildRootID: uint64(i), Filename: args.KojiFilenames[i], FileSize: kojiTargetOptions.ImageSize, Arch: buildArgs.Arch, - ChecksumType: "md5", - MD5: kojiTargetOptions.ImageMD5, - Type: "image", + ChecksumType: koji.ChecksumTypeMD5, + Checksum: kojiTargetOptions.ImageMD5, + Type: koji.BuildOutputTypeImage, RPMs: imageRPMs, - Extra: koji.ImageExtra{ - Info: koji.ImageExtraInfo{ + Extra: koji.BuildOutputExtra{ + Image: koji.ImageExtraInfo{ Arch: buildArgs.Arch, }, }, }) } - err = impl.kojiImport(args.Server, build, buildRoots, images, args.KojiDirectory, initArgs.Token) + err = impl.kojiImport(args.Server, build, buildRoots, outputs, args.KojiDirectory, initArgs.Token) if err != nil { kojiFinalizeJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorKojiFinalize, err.Error(), nil) return err diff --git a/internal/upload/koji/koji.go b/internal/upload/koji/koji.go index bb9695e88..1659accf7 100644 --- a/internal/upload/koji/koji.go +++ b/internal/upload/koji/koji.go @@ -3,6 +3,7 @@ package koji import ( "bytes" "context" + // koji uses MD5 hashes /* #nosec G501 */ "crypto/md5" @@ -34,46 +35,63 @@ type Koji struct { transport http.RoundTripper } +// BUILD METADATA + +// TypeInfo is a map whose entries are the names of the build types +// used for the build, and the values are free-form maps containing +// type-specific information for the build. type TypeInfo struct { Image struct{} `json:"image"` } -type ImageBuildExtra struct { +// BuildExtra holds extra metadata associated with the build. +// It is a free-form map, but must contain at least the 'typeinfo' key. +type BuildExtra struct { TypeInfo TypeInfo `json:"typeinfo"` } -type ImageBuild struct { - BuildID uint64 `json:"build_id"` - TaskID uint64 `json:"task_id"` - Name string `json:"name"` - Version string `json:"version"` - Release string `json:"release"` - Source string `json:"source"` - StartTime int64 `json:"start_time"` - EndTime int64 `json:"end_time"` - Extra ImageBuildExtra `json:"extra"` +// Build represents a Koji build and holds metadata about it. +type Build struct { + BuildID uint64 `json:"build_id"` + TaskID uint64 `json:"task_id"` + Name string `json:"name"` + Version string `json:"version"` + Release string `json:"release"` + Source string `json:"source"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + // NOTE: This is the struct that ends up shown in the buildinfo and webui in Koji. + Extra BuildExtra `json:"extra"` } +// BUIDROOT METADATA + +// Host holds information about the host where the build was run. type Host struct { Os string `json:"os"` Arch string `json:"arch"` } +// ContentGenerator holds information about the content generator which run the build. type ContentGenerator struct { Name string `json:"name"` // Must be 'osbuild'. Version string `json:"version"` } +// Container holds information about the container in which the build was run. type Container struct { + // Type of the container that was used, e.g. 'none', 'chroot', 'kvm', 'docker', etc. Type string `json:"type"` Arch string `json:"arch"` } +// Tool holds information about a tool used to run build. type Tool struct { Name string `json:"name"` Version string `json:"version"` } +// BuildRoot represents a buildroot used for the build. type BuildRoot struct { ID uint64 `json:"id"` Host Host `json:"host"` @@ -83,34 +101,65 @@ type BuildRoot struct { RPMs []rpmmd.RPM `json:"components"` } +// OUTPUT METADATA + +// ImageExtraInfo holds extra metadata about the image. +// This structure is shared for the Extra metadata of the output and the build. type ImageExtraInfo struct { // TODO: Ideally this is where the pipeline would be passed. Arch string `json:"arch"` // TODO: why? } -type ImageExtra struct { - Info ImageExtraInfo `json:"image"` +// BuildOutputExtra holds extra metadata associated with the build output. +type BuildOutputExtra struct { + Image ImageExtraInfo `json:"image"` } -type Image struct { - BuildRootID uint64 `json:"buildroot_id"` - Filename string `json:"filename"` - FileSize uint64 `json:"filesize"` - Arch string `json:"arch"` - ChecksumType string `json:"checksum_type"` // must be 'md5' - MD5 string `json:"checksum"` - Type string `json:"type"` - RPMs []rpmmd.RPM `json:"components"` - Extra ImageExtra `json:"extra"` +// BuildOutputType represents the type of a BuildOutput. +type BuildOutputType string + +const ( + BuildOutputTypeImage BuildOutputType = "image" +) + +// ChecksumType represents the type of a checksum used for a BuildOutput. +type ChecksumType string + +const ( + ChecksumTypeMD5 ChecksumType = "md5" + ChecksumTypeAdler32 ChecksumType = "adler32" + ChecksumTypeSHA256 ChecksumType = "sha256" +) + +// BuildOutput represents an output from the OSBuild content generator. +// The output can be a file of various types, which is imported to Koji. +// Examples of types are "image", "log" or other. +type BuildOutput struct { + BuildRootID uint64 `json:"buildroot_id"` + Filename string `json:"filename"` + FileSize uint64 `json:"filesize"` + Arch string `json:"arch"` // can be 'noarch' or a specific arch + ChecksumType ChecksumType `json:"checksum_type"` + Checksum string `json:"checksum"` + Type BuildOutputType `json:"type"` + RPMs []rpmmd.RPM `json:"components"` // TODO: should be omitempty + Extra BuildOutputExtra `json:"extra"` // TODO: should be omitempty } +// CONTENT GENERATOR METADATA + +// Metadata holds Koji Content Generator metadata. +// This is passed to the CGImport call. +// For more information, see https://docs.pagure.org/koji/content_generator_metadata/ type Metadata struct { - MetadataVersion int `json:"metadata_version"` // must be '0' - ImageBuild ImageBuild `json:"build"` - BuildRoots []BuildRoot `json:"buildroots"` - Images []Image `json:"output"` + MetadataVersion int `json:"metadata_version"` // must be '0' + Build Build `json:"build"` + BuildRoots []BuildRoot `json:"buildroots"` + Outputs []BuildOutput `json:"output"` } +// KOJI API STRUCTURES + type CGInitBuildResult struct { BuildID int `xmlrpc:"build_id"` Token string `xmlrpc:"token"` @@ -276,11 +325,11 @@ func (k *Koji) CGCancelBuild(buildID int, token string) error { // CGImport imports previously uploaded content, by specifying its metadata, and the temporary // directory where it is located. -func (k *Koji) CGImport(build ImageBuild, buildRoots []BuildRoot, images []Image, directory, token string) (*CGImportResult, error) { +func (k *Koji) CGImport(build Build, buildRoots []BuildRoot, outputs []BuildOutput, directory, token string) (*CGImportResult, error) { m := &Metadata{ - ImageBuild: build, + Build: build, BuildRoots: buildRoots, - Images: images, + Outputs: outputs, } metadata, err := json.Marshal(m) if err != nil {