particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Koji: refactor and consolidate structures

Browse source 
The original Koji implementation expected that the output of a content
generator is only an image. While in reality, we will eventually upload
other types of files as outputs to Koji, such as logs and osbuild
manifest.

Rename Koji structures and their members to better map to the upstream
Koji documentation and their JSON representation. Add comments to
structures. Define type aliases and constants for string values which
are more like enums, than a free-form values.

These changes have no effect on the actual JSON representation of any of
the structures

Signed-off-by: Tomáš Hozza <thozza@redhat.com>
This commit is contained in:
Tomáš Hozza 2023-08-01 17:23:33 +02:00 • committed by Ondřej Budai
parent 1b04a78120
commit 350762497f
4 changed files with 105 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

14
cmd/osbuild-koji-tests/main_test.go
View file

@ -141,7 +141,7 @@ func TestKojiImport(t *testing.T) {
require.NoError(t, err)
// Import the build
build := koji.ImageBuild{
build := koji.Build{
TaskID: 1,
Name: buildName,
Version: "1",
@ -168,18 +168,18 @@ func TestKojiImport(t *testing.T) {
RPMs: []rpmmd.RPM{},
},
}
output := []koji.Image{
output := []koji.BuildOutput{
{
BuildRootID: 1,
Filename: filename,
FileSize: uint64(filesize),
Arch: "noarch",
ChecksumType: "md5",
MD5: hash,
Type: "image",
ChecksumType: koji.ChecksumTypeMD5,
Checksum: hash,
Type: koji.BuildOutputTypeImage,
RPMs: []rpmmd.RPM{},
Extra: koji.ImageExtra{
Info: koji.ImageExtraInfo{
Extra: koji.BuildOutputExtra{
Image: koji.ImageExtraInfo{
Arch: "noarch",
},
},

14
cmd/osbuild-koji/main.go
View file

@ -60,7 +60,7 @@ func main() {
return
}
build := koji.ImageBuild{
build := koji.Build{
TaskID: uint64(taskID),
Name: name,
Version: version,
@ -87,18 +87,18 @@ func main() {
RPMs: []rpmmd.RPM{},
},
}
output := []koji.Image{
output := []koji.BuildOutput{
{
BuildRootID: 1,
Filename: path.Base(filename),
FileSize: length,
Arch: arch,
ChecksumType: "md5",
MD5: hash,
Type: "image",
ChecksumType: koji.ChecksumTypeMD5,
Checksum: hash,
Type: koji.BuildOutputTypeImage,
RPMs: []rpmmd.RPM{},
Extra: koji.ImageExtra{
Info: koji.ImageExtraInfo{
Extra: koji.BuildOutputExtra{
Image: koji.ImageExtraInfo{
Arch: arch,
},
},

24
cmd/osbuild-worker/jobimpl-koji-finalize.go
View file

@ -21,9 +21,9 @@ type KojiFinalizeJobImpl struct {
func (impl *KojiFinalizeJobImpl) kojiImport(
server string,
build koji.ImageBuild,
build koji.Build,
buildRoots []koji.BuildRoot,
images []koji.Image,
outputs []koji.BuildOutput,
directory, token string) error {
serverURL, err := url.Parse(server)
@ -48,7 +48,7 @@ func (impl *KojiFinalizeJobImpl) kojiImport(
}
}()
_, err = k.CGImport(build, buildRoots, images, directory, token)
_, err = k.CGImport(build, buildRoots, outputs, directory, token)
if err != nil {
return fmt.Errorf("Could not import build into koji: %v", err)
}
@ -115,7 +115,7 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error {
return err
}
build := koji.ImageBuild{
build := koji.Build{
TaskID: args.TaskID,
Name: args.Name,
Version: args.Version,
@ -125,7 +125,7 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error {
}
var buildRoots []koji.BuildRoot
var images []koji.Image
var outputs []koji.BuildOutput
var osbuildResults []worker.OSBuildJobResult
initArgs, osbuildResults, err = extractDynamicArgs(job)
@ -191,24 +191,24 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error {
// deduplicate
imageRPMs = rpmmd.DeduplicateRPMs(imageRPMs)
images = append(images, koji.Image{
outputs = append(outputs, koji.BuildOutput{
BuildRootID: uint64(i),
Filename: args.KojiFilenames[i],
FileSize: kojiTargetOptions.ImageSize,
Arch: buildArgs.Arch,
ChecksumType: "md5",
MD5: kojiTargetOptions.ImageMD5,
Type: "image",
ChecksumType: koji.ChecksumTypeMD5,
Checksum: kojiTargetOptions.ImageMD5,
Type: koji.BuildOutputTypeImage,
RPMs: imageRPMs,
Extra: koji.ImageExtra{
Info: koji.ImageExtraInfo{
Extra: koji.BuildOutputExtra{
Image: koji.ImageExtraInfo{
Arch: buildArgs.Arch,
},
},
})
}
err = impl.kojiImport(args.Server, build, buildRoots, images, args.KojiDirectory, initArgs.Token)
err = impl.kojiImport(args.Server, build, buildRoots, outputs, args.KojiDirectory, initArgs.Token)
if err != nil {
kojiFinalizeJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorKojiFinalize, err.Error(), nil)
return err

109
internal/upload/koji/koji.go
View file

@ -3,6 +3,7 @@ package koji
import (
"bytes"
"context"
// koji uses MD5 hashes
/* #nosec G501 */
"crypto/md5"
@ -34,46 +35,63 @@ type Koji struct {
transport http.RoundTripper
}
// BUILD METADATA
// TypeInfo is a map whose entries are the names of the build types
// used for the build, and the values are free-form maps containing
// type-specific information for the build.
type TypeInfo struct {
Image struct{} `json:"image"`
}
type ImageBuildExtra struct {
// BuildExtra holds extra metadata associated with the build.
// It is a free-form map, but must contain at least the 'typeinfo' key.
type BuildExtra struct {
TypeInfo TypeInfo `json:"typeinfo"`
}
type ImageBuild struct {
BuildID uint64 `json:"build_id"`
TaskID uint64 `json:"task_id"`
Name string `json:"name"`
Version string `json:"version"`
Release string `json:"release"`
Source string `json:"source"`
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
Extra ImageBuildExtra `json:"extra"`
// Build represents a Koji build and holds metadata about it.
type Build struct {
BuildID uint64 `json:"build_id"`
TaskID uint64 `json:"task_id"`
Name string `json:"name"`
Version string `json:"version"`
Release string `json:"release"`
Source string `json:"source"`
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
// NOTE: This is the struct that ends up shown in the buildinfo and webui in Koji.
Extra BuildExtra `json:"extra"`
}
// BUIDROOT METADATA
// Host holds information about the host where the build was run.
type Host struct {
Os string `json:"os"`
Arch string `json:"arch"`
}
// ContentGenerator holds information about the content generator which run the build.
type ContentGenerator struct {
Name string `json:"name"` // Must be 'osbuild'.
Version string `json:"version"`
}
// Container holds information about the container in which the build was run.
type Container struct {
// Type of the container that was used, e.g. 'none', 'chroot', 'kvm', 'docker', etc.
Type string `json:"type"`
Arch string `json:"arch"`
}
// Tool holds information about a tool used to run build.
type Tool struct {
Name string `json:"name"`
Version string `json:"version"`
}
// BuildRoot represents a buildroot used for the build.
type BuildRoot struct {
ID uint64 `json:"id"`
Host Host `json:"host"`
@ -83,34 +101,65 @@ type BuildRoot struct {
RPMs []rpmmd.RPM `json:"components"`
}
// OUTPUT METADATA
// ImageExtraInfo holds extra metadata about the image.
// This structure is shared for the Extra metadata of the output and the build.
type ImageExtraInfo struct {
// TODO: Ideally this is where the pipeline would be passed.
Arch string `json:"arch"` // TODO: why?
}
type ImageExtra struct {
Info ImageExtraInfo `json:"image"`
// BuildOutputExtra holds extra metadata associated with the build output.
type BuildOutputExtra struct {
Image ImageExtraInfo `json:"image"`
}
type Image struct {
BuildRootID uint64 `json:"buildroot_id"`
Filename string `json:"filename"`
FileSize uint64 `json:"filesize"`
Arch string `json:"arch"`
ChecksumType string `json:"checksum_type"` // must be 'md5'
MD5 string `json:"checksum"`
Type string `json:"type"`
RPMs []rpmmd.RPM `json:"components"`
Extra ImageExtra `json:"extra"`
// BuildOutputType represents the type of a BuildOutput.
type BuildOutputType string
const (
BuildOutputTypeImage BuildOutputType = "image"
)
// ChecksumType represents the type of a checksum used for a BuildOutput.
type ChecksumType string
const (
ChecksumTypeMD5 ChecksumType = "md5"
ChecksumTypeAdler32 ChecksumType = "adler32"
ChecksumTypeSHA256 ChecksumType = "sha256"
)
// BuildOutput represents an output from the OSBuild content generator.
// The output can be a file of various types, which is imported to Koji.
// Examples of types are "image", "log" or other.
type BuildOutput struct {
BuildRootID uint64 `json:"buildroot_id"`
Filename string `json:"filename"`
FileSize uint64 `json:"filesize"`
Arch string `json:"arch"` // can be 'noarch' or a specific arch
ChecksumType ChecksumType `json:"checksum_type"`
Checksum string `json:"checksum"`
Type BuildOutputType `json:"type"`
RPMs []rpmmd.RPM `json:"components"` // TODO: should be omitempty
Extra BuildOutputExtra `json:"extra"` // TODO: should be omitempty
}
// CONTENT GENERATOR METADATA
// Metadata holds Koji Content Generator metadata.
// This is passed to the CGImport call.
// For more information, see https://docs.pagure.org/koji/content_generator_metadata/
type Metadata struct {
MetadataVersion int `json:"metadata_version"` // must be '0'
ImageBuild ImageBuild `json:"build"`
BuildRoots []BuildRoot `json:"buildroots"`
Images []Image `json:"output"`
MetadataVersion int `json:"metadata_version"` // must be '0'
Build Build `json:"build"`
BuildRoots []BuildRoot `json:"buildroots"`
Outputs []BuildOutput `json:"output"`
}
// KOJI API STRUCTURES
type CGInitBuildResult struct {
BuildID int `xmlrpc:"build_id"`
Token string `xmlrpc:"token"`
@ -276,11 +325,11 @@ func (k *Koji) CGCancelBuild(buildID int, token string) error {
// CGImport imports previously uploaded content, by specifying its metadata, and the temporary
// directory where it is located.
func (k *Koji) CGImport(build ImageBuild, buildRoots []BuildRoot, images []Image, directory, token string) (*CGImportResult, error) {
func (k *Koji) CGImport(build Build, buildRoots []BuildRoot, outputs []BuildOutput, directory, token string) (*CGImportResult, error) {
m := &Metadata{
ImageBuild: build,
Build: build,
BuildRoots: buildRoots,
Images: images,
Outputs: outputs,
}
metadata, err := json.Marshal(m)
if err != nil {