Many: move to koji upload implementation from osbuild/images
Delete the `internal/upload/koji` package and replace it with `pkg/upload/koji` package provided by `osbuild/images`. Signed-off-by: Tomáš Hozza <thozza@redhat.com>
This commit is contained in:
Tomáš Hozza
Sanne Raymaekers
parent
7a580f79ae
commit
3e3f9a0789
11 changed files with 8 additions and 215 deletions
|
|
@ -1,418 +0,0 @@
|
|||
//go:build cgo
|
||||
|
||||
// koji requires the khttp kerberos module which requires cgo so when
|
||||
// build without cgo kerberos uploads are currently not supported
|
||||
|
||||
package koji
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
|
||||
// koji uses MD5 hashes
|
||||
/* #nosec G501 */
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/adler32"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
rh "github.com/hashicorp/go-retryablehttp"
|
||||
"github.com/kolo/xmlrpc"
|
||||
"github.com/ubccr/kerby/khttp"
|
||||
)
|
||||
|
||||
type Koji struct {
|
||||
xmlrpc *xmlrpc.Client
|
||||
server string
|
||||
transport http.RoundTripper
|
||||
logger rh.LeveledLogger
|
||||
}
|
||||
|
||||
// KOJI API STRUCTURES
|
||||
|
||||
type CGInitBuildResult struct {
|
||||
BuildID int `xmlrpc:"build_id"`
|
||||
Token string `xmlrpc:"token"`
|
||||
}
|
||||
|
||||
type CGImportResult struct {
|
||||
BuildID int `xmlrpc:"build_id"`
|
||||
}
|
||||
|
||||
type GSSAPICredentials struct {
|
||||
Principal string
|
||||
KeyTab string
|
||||
}
|
||||
|
||||
type loginReply struct {
|
||||
SessionID int64 `xmlrpc:"session-id"`
|
||||
SessionKey string `xmlrpc:"session-key"`
|
||||
}
|
||||
|
||||
func newKoji(server string, transport http.RoundTripper, reply loginReply, logger rh.LeveledLogger) (*Koji, error) {
|
||||
// Create the final xmlrpc client with our custom RoundTripper handling
|
||||
// sessionID, sessionKey and callnum
|
||||
kojiTransport := &Transport{
|
||||
sessionID: reply.SessionID,
|
||||
sessionKey: reply.SessionKey,
|
||||
callnum: 0,
|
||||
transport: transport,
|
||||
}
|
||||
|
||||
client, err := xmlrpc.NewClient(server, kojiTransport)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Koji{
|
||||
xmlrpc: client,
|
||||
server: server,
|
||||
transport: kojiTransport,
|
||||
logger: logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewFromGSSAPI creates a new Koji session authenticated using GSSAPI.
|
||||
// Principal and keytab used for the session is passed using credentials
|
||||
// parameter.
|
||||
func NewFromGSSAPI(
|
||||
server string,
|
||||
credentials *GSSAPICredentials,
|
||||
transport http.RoundTripper,
|
||||
logger rh.LeveledLogger) (*Koji, error) {
|
||||
// Create a temporary xmlrpc client with kerberos transport.
|
||||
// The API doesn't require sessionID, sessionKey and callnum yet,
|
||||
// so there's no need to use the custom Koji RoundTripper,
|
||||
// let's just use the one that the called passed in.
|
||||
loginClient, err := xmlrpc.NewClient(server+"/ssllogin", &khttp.Transport{
|
||||
KeyTab: credentials.KeyTab,
|
||||
Principal: credentials.Principal,
|
||||
Next: transport,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var reply loginReply
|
||||
err = loginClient.Call("sslLogin", nil, &reply)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return newKoji(server, transport, reply, logger)
|
||||
}
|
||||
|
||||
// GetAPIVersion gets the version of the API of the remote Koji instance
|
||||
func (k *Koji) GetAPIVersion() (int, error) {
|
||||
var version int
|
||||
err := k.xmlrpc.Call("getAPIVersion", nil, &version)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// Logout ends the session
|
||||
func (k *Koji) Logout() error {
|
||||
err := k.xmlrpc.Call("logout", nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CGInitBuild reserves a build ID and initializes a build
|
||||
func (k *Koji) CGInitBuild(name, version, release string) (*CGInitBuildResult, error) {
|
||||
var buildInfo struct {
|
||||
Name string `xmlrpc:"name"`
|
||||
Version string `xmlrpc:"version"`
|
||||
Release string `xmlrpc:"release"`
|
||||
}
|
||||
|
||||
buildInfo.Name = name
|
||||
buildInfo.Version = version
|
||||
buildInfo.Release = release
|
||||
|
||||
var result CGInitBuildResult
|
||||
err := k.xmlrpc.Call("CGInitBuild", []interface{}{"osbuild", buildInfo}, &result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
/*
|
||||
from `koji/__init__.py`
|
||||
|
||||
BUILD_STATES = Enum((
|
||||
|
||||
'BUILDING',
|
||||
'COMPLETE',
|
||||
'DELETED',
|
||||
'FAILED',
|
||||
'CANCELED',
|
||||
|
||||
))
|
||||
*/
|
||||
const (
|
||||
_ = iota /* BUILDING */
|
||||
_ /* COMPLETED */
|
||||
_ /* DELETED */
|
||||
buildStateFailed
|
||||
buildStateCanceled
|
||||
)
|
||||
|
||||
// CGFailBuild marks an in-progress build as failed
|
||||
func (k *Koji) CGFailBuild(buildID int, token string) error {
|
||||
return k.xmlrpc.Call("CGRefundBuild", []interface{}{"osbuild", buildID, token, buildStateFailed}, nil)
|
||||
}
|
||||
|
||||
// CGCancelBuild marks an in-progress build as cancelled, and
|
||||
func (k *Koji) CGCancelBuild(buildID int, token string) error {
|
||||
return k.xmlrpc.Call("CGRefundBuild", []interface{}{"osbuild", buildID, token, buildStateCanceled}, nil)
|
||||
}
|
||||
|
||||
// CGImport imports previously uploaded content, by specifying its metadata, and the temporary
|
||||
// directory where it is located.
|
||||
func (k *Koji) CGImport(build Build, buildRoots []BuildRoot, outputs []BuildOutput, directory, token string) (*CGImportResult, error) {
|
||||
m := &Metadata{
|
||||
Build: build,
|
||||
BuildRoots: buildRoots,
|
||||
Outputs: outputs,
|
||||
}
|
||||
metadata, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
const retryCount = 10
|
||||
const retryDelay = time.Second
|
||||
|
||||
for attempt := 0; attempt < retryCount; attempt += 1 {
|
||||
var result CGImportResult
|
||||
err = k.xmlrpc.Call("CGImport", []interface{}{string(metadata), directory, token}, &result)
|
||||
|
||||
if err != nil {
|
||||
// Retry when the error mentions a corrupted upload. It's usually
|
||||
// just because of NFS inconsistency when the kojihub has multiple
|
||||
// replicas.
|
||||
if strings.Contains(err.Error(), "Corrupted upload") {
|
||||
time.Sleep(retryDelay)
|
||||
continue
|
||||
}
|
||||
|
||||
// Fail immediately on other errors, they are probably legitimate
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if k.logger != nil {
|
||||
k.logger.Info(fmt.Sprintf("CGImport succeeded after %d attempts", attempt+1))
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to import a build after %d attempts: %w", retryCount, err)
|
||||
}
|
||||
|
||||
// uploadChunk uploads a byte slice to a given filepath/filname at a given offset
|
||||
func (k *Koji) uploadChunk(chunk []byte, filepath, filename string, offset uint64) error {
|
||||
// We have to open-code a bastardized version of XML-RPC: We send an octet-stream, as
|
||||
// if it was an RPC call, and get a regular XML-RPC reply back. In addition to the
|
||||
// standard URL parameters, we also have to pass any other parameters as part of the
|
||||
// URL, as the body can only contain the payload.
|
||||
u, err := url.Parse(k.server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q := u.Query()
|
||||
q.Add("filepath", filepath)
|
||||
q.Add("filename", filename)
|
||||
q.Add("offset", fmt.Sprintf("%v", offset))
|
||||
q.Add("fileverify", "adler32")
|
||||
q.Add("overwrite", "true")
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
client := createCustomRetryableClient(k.logger)
|
||||
|
||||
client.HTTPClient = &http.Client{
|
||||
Transport: k.transport,
|
||||
}
|
||||
|
||||
respData, err := client.Post(u.String(), "application/octet-stream", bytes.NewBuffer(chunk))
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer respData.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(respData.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var reply struct {
|
||||
Size int `xmlrpc:"size"`
|
||||
HexDigest string `xmlrpc:"hexdigest"`
|
||||
}
|
||||
|
||||
resp := xmlrpc.Response(body)
|
||||
|
||||
if resp.Err() != nil {
|
||||
return fmt.Errorf("xmlrpc server returned an error: %v", resp.Err())
|
||||
}
|
||||
|
||||
err = resp.Unmarshal(&reply)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot unmarshal the xmlrpc response: %v", err)
|
||||
}
|
||||
|
||||
if reply.Size != len(chunk) {
|
||||
return fmt.Errorf("Sent a chunk of %d bytes, but server got %d bytes", len(chunk), reply.Size)
|
||||
}
|
||||
|
||||
digest := fmt.Sprintf("%08x", adler32.Checksum(chunk))
|
||||
if reply.HexDigest != digest {
|
||||
return fmt.Errorf("Sent a chunk with Adler32 digest %s, but server computed digest %s", digest, reply.HexDigest)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Upload uploads file to the temporary filepath on the kojiserver under the name filename
|
||||
// The md5sum and size of the file is returned on success.
|
||||
func (k *Koji) Upload(file io.Reader, filepath, filename string) (string, uint64, error) {
|
||||
chunk := make([]byte, 1024*1024) // upload a megabyte at a time
|
||||
offset := uint64(0)
|
||||
// Koji uses MD5 hashes
|
||||
/* #nosec G401 */
|
||||
hash := md5.New()
|
||||
for {
|
||||
n, err := file.Read(chunk)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return "", 0, err
|
||||
}
|
||||
err = k.uploadChunk(chunk[:n], filepath, filename, offset)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
// NB: the 'n' returned by Read() of an io.Reader can never be negative
|
||||
/* #nosec G115 */
|
||||
offset += uint64(n)
|
||||
|
||||
m, err := hash.Write(chunk[:n])
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if m != n {
|
||||
return "", 0, fmt.Errorf("sent %d bytes, but hashed %d", n, m)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%x", hash.Sum(nil)), offset, nil
|
||||
}
|
||||
|
||||
type Transport struct {
|
||||
sessionID int64
|
||||
sessionKey string
|
||||
callnum int
|
||||
transport http.RoundTripper
|
||||
}
|
||||
|
||||
// RoundTrip implements the RoundTripper interface, using the default
|
||||
// transport. When a session has been established, also pass along the
|
||||
// session credentials. This may not be how the RoundTripper interface
|
||||
// is meant to be used, but the underlying XML-RPC helpers don't allow
|
||||
// us to adjust the URL per-call (these arguments should really be in
|
||||
// the body).
|
||||
func (rt *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
// Clone the request, so as not to alter the passed in value.
|
||||
rClone := new(http.Request)
|
||||
*rClone = *req
|
||||
rClone.Header = make(http.Header, len(req.Header))
|
||||
for idx, header := range req.Header {
|
||||
rClone.Header[idx] = append([]string(nil), header...)
|
||||
}
|
||||
|
||||
values := rClone.URL.Query()
|
||||
values.Add("session-id", fmt.Sprintf("%v", rt.sessionID))
|
||||
values.Add("session-key", rt.sessionKey)
|
||||
values.Add("callnum", fmt.Sprintf("%v", rt.callnum))
|
||||
rClone.URL.RawQuery = values.Encode()
|
||||
|
||||
// Each call is given a unique callnum.
|
||||
rt.callnum++
|
||||
|
||||
return rt.transport.RoundTrip(rClone)
|
||||
}
|
||||
|
||||
func CreateKojiTransport(relaxTimeout time.Duration, logger rh.LeveledLogger) http.RoundTripper {
|
||||
// Koji for some reason needs TLS renegotiation enabled.
|
||||
// Clone the default http rt and enable renegotiation.
|
||||
rt := CreateRetryableTransport(logger)
|
||||
|
||||
transport := rt.Client.HTTPClient.Transport.(*http.Transport)
|
||||
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
Renegotiation: tls.RenegotiateOnceAsClient,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}
|
||||
|
||||
// Relax timeouts a bit
|
||||
if relaxTimeout > 0 {
|
||||
transport.TLSHandshakeTimeout *= relaxTimeout
|
||||
transport.DialContext = (&net.Dialer{
|
||||
Timeout: 30 * time.Second * relaxTimeout,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext
|
||||
}
|
||||
|
||||
return rt
|
||||
}
|
||||
|
||||
func createCustomRetryableClient(logger rh.LeveledLogger) *rh.Client {
|
||||
client := rh.NewClient()
|
||||
client.Logger = logger
|
||||
|
||||
client.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
shouldRetry, retErr := rh.DefaultRetryPolicy(ctx, resp, err)
|
||||
|
||||
// DefaultRetryPolicy denies retrying for any certificate related error.
|
||||
// Override it in case the error is a timeout.
|
||||
if !shouldRetry && err != nil {
|
||||
if v, ok := err.(*url.Error); ok {
|
||||
if _, ok := v.Err.(x509.UnknownAuthorityError); ok {
|
||||
// retry if it's a timeout
|
||||
return strings.Contains(strings.ToLower(v.Error()), "timeout"), v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if logger != nil && (!shouldRetry && !(resp.StatusCode >= 200 && resp.StatusCode < 300)) {
|
||||
logger.Info(fmt.Sprintf("Not retrying: %v", resp.Status))
|
||||
}
|
||||
|
||||
return shouldRetry, retErr
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func CreateRetryableTransport(logger rh.LeveledLogger) *rh.RoundTripper {
|
||||
rt := rh.RoundTripper{}
|
||||
rt.Client = createCustomRetryableClient(logger)
|
||||
return &rt
|
||||
}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
package koji
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/osbuild/images/pkg/osbuild"
|
||||
)
|
||||
|
||||
// RPM represents an RPM package in the Koji metadata format.
|
||||
// It contains the necessary fields to uniquely identify an RPM package,
|
||||
// when desdribing the build metadata in Koji.
|
||||
type RPM struct {
|
||||
Type string `json:"type"` // must be 'rpm'
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Release string `json:"release"`
|
||||
Epoch *string `json:"epoch,omitempty"`
|
||||
Arch string `json:"arch"`
|
||||
Sigmd5 string `json:"sigmd5"`
|
||||
Signature *string `json:"signature"`
|
||||
}
|
||||
|
||||
// NEVRA string for the package
|
||||
func (r RPM) String() string {
|
||||
epoch := ""
|
||||
if r.Epoch != nil {
|
||||
epoch = *r.Epoch + ":"
|
||||
}
|
||||
return fmt.Sprintf("%s-%s%s-%s.%s", r.Name, epoch, r.Version, r.Release, r.Arch)
|
||||
}
|
||||
|
||||
// Deduplicate a list of RPMs based on NEVRA string
|
||||
func DeduplicateRPMs(rpms []RPM) []RPM {
|
||||
rpmMap := make(map[string]struct{}, len(rpms))
|
||||
uniqueRPMs := make([]RPM, 0, len(rpms))
|
||||
|
||||
for _, rpm := range rpms {
|
||||
if _, added := rpmMap[rpm.String()]; !added {
|
||||
rpmMap[rpm.String()] = struct{}{}
|
||||
uniqueRPMs = append(uniqueRPMs, rpm)
|
||||
}
|
||||
}
|
||||
return uniqueRPMs
|
||||
}
|
||||
|
||||
func OSBuildMetadataToRPMs(stagesMetadata map[string]osbuild.StageMetadata) []RPM {
|
||||
rpms := make([]RPM, 0)
|
||||
for _, md := range stagesMetadata {
|
||||
switch metadata := md.(type) {
|
||||
case *osbuild.RPMStageMetadata:
|
||||
for _, pkg := range metadata.Packages {
|
||||
rpms = append(rpms, RPM{
|
||||
Type: "rpm",
|
||||
Name: pkg.Name,
|
||||
Epoch: pkg.Epoch,
|
||||
Version: pkg.Version,
|
||||
Release: pkg.Release,
|
||||
Arch: pkg.Arch,
|
||||
Sigmd5: pkg.SigMD5,
|
||||
Signature: pkg.Signature(),
|
||||
})
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
return rpms
|
||||
}
|
||||
|
|
@ -1,208 +0,0 @@
|
|||
package koji
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/osbuild/images/pkg/osbuild"
|
||||
"github.com/osbuild/osbuild-composer/internal/common"
|
||||
)
|
||||
|
||||
func TestRPMDeduplication(t *testing.T) {
|
||||
require := require.New(t)
|
||||
// start with metadata, that includes duplicates, convert, then deduplicate
|
||||
metadata := osbuild.PipelineMetadata{
|
||||
"1": &osbuild.RPMStageMetadata{
|
||||
Packages: []osbuild.RPMPackageMetadata{
|
||||
// python38 twice
|
||||
{
|
||||
Name: "python38",
|
||||
Version: "3.8.8",
|
||||
Release: "4.module+el8.5.0+12205+a865257a",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
SigMD5: "-",
|
||||
SigPGP: "-",
|
||||
SigGPG: "-",
|
||||
},
|
||||
{
|
||||
Name: "python38",
|
||||
Version: "3.8.8",
|
||||
Release: "4.module+el8.5.0+12205+a865257a",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
SigMD5: "-",
|
||||
SigPGP: "-",
|
||||
SigGPG: "-",
|
||||
},
|
||||
// made up package
|
||||
{
|
||||
Name: "unique",
|
||||
Version: "1.90",
|
||||
Release: "10",
|
||||
Epoch: nil,
|
||||
Arch: "aarch64",
|
||||
SigMD5: ".",
|
||||
SigPGP: ".",
|
||||
SigGPG: ".",
|
||||
},
|
||||
// made up package with epoch
|
||||
{
|
||||
Name: "package-with-epoch",
|
||||
Version: "0.1",
|
||||
Release: "a",
|
||||
Epoch: common.ToPtr("8"),
|
||||
Arch: "x86_64",
|
||||
SigMD5: "*",
|
||||
SigPGP: "*",
|
||||
SigGPG: "*",
|
||||
},
|
||||
},
|
||||
},
|
||||
// separate pipeline
|
||||
"2": &osbuild.RPMStageMetadata{
|
||||
Packages: []osbuild.RPMPackageMetadata{
|
||||
// duplicate package with epoch
|
||||
{
|
||||
Name: "vim-minimal",
|
||||
Version: "8.0.1763",
|
||||
Release: "15.el8",
|
||||
Epoch: common.ToPtr("2"),
|
||||
Arch: "x86_64",
|
||||
SigMD5: "v",
|
||||
SigPGP: "v",
|
||||
SigGPG: "v",
|
||||
},
|
||||
{
|
||||
Name: "vim-minimal",
|
||||
Version: "8.0.1763",
|
||||
Release: "15.el8",
|
||||
Epoch: common.ToPtr("2"),
|
||||
Arch: "x86_64",
|
||||
SigMD5: "v",
|
||||
SigPGP: "v",
|
||||
SigGPG: "v",
|
||||
},
|
||||
// package with same name but different version
|
||||
{
|
||||
Name: "dupename",
|
||||
Version: "1",
|
||||
Release: "1.el8",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
SigMD5: "2",
|
||||
SigPGP: "2",
|
||||
SigGPG: "2",
|
||||
},
|
||||
{
|
||||
Name: "dupename",
|
||||
Version: "2",
|
||||
Release: "1.el8",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
SigMD5: "2",
|
||||
SigPGP: "2",
|
||||
SigGPG: "2",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
testNames := []string{"dupename", "dupename", "package-with-epoch", "python38", "python38", "unique", "vim-minimal", "vim-minimal"}
|
||||
testNamesDeduped := []string{"dupename", "dupename", "package-with-epoch", "python38", "unique", "vim-minimal"}
|
||||
|
||||
rpms := OSBuildMetadataToRPMs(metadata)
|
||||
|
||||
// basic sanity checks
|
||||
require.Len(rpms, 8)
|
||||
|
||||
sortedNames := func(rpms []RPM) []string {
|
||||
names := make([]string, len(rpms))
|
||||
for idx, rpm := range rpms {
|
||||
names[idx] = rpm.Name
|
||||
}
|
||||
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
names := sortedNames(rpms)
|
||||
require.Equal(names, testNames)
|
||||
|
||||
deduped := DeduplicateRPMs(rpms)
|
||||
require.Len(deduped, 6)
|
||||
dedupedNames := sortedNames(deduped)
|
||||
require.Equal(dedupedNames, testNamesDeduped)
|
||||
}
|
||||
|
||||
func Test_OSBuildMetadataToRPMs(t *testing.T) {
|
||||
raw := `
|
||||
{
|
||||
"org.osbuild.rpm": {
|
||||
"packages": [
|
||||
{
|
||||
"name": "python3-pyserial",
|
||||
"version": "3.4",
|
||||
"release": "7.fc32",
|
||||
"epoch": null,
|
||||
"arch": "noarch",
|
||||
"sigmd5": "378cb32f9f850b275ac4e04d21e8144b",
|
||||
"sigpgp": "89023304000108001d162104963a2beb02009608fe67ea4249fd77499570ff3105025f5a272b000a091049fd77499570ff31ccdb0ffe38b95a55ebf3c021526b3cd4f2358c7e23f7767d1f5ce4b7cccef7b33653c6a96a23022313a818fbaf7abeb41837910f0d3ac15664e02838d5939d38ff459aa0076e248728a032d3ae09ddfaec955f941601081a2e3f9bbd49586fd65c1bc1b31685aeb0405687d1791471eab7359ccf00d5584ddef680e99ebc8a4846316391b9baa68ac8ed8ad696ee16fd625d847f8edd92517df3ea6920a46b77b4f119715a0f619f38835d25e0bd0eb5cfad08cd9c796eace6a2b28f4d3dee552e6068255d9748dc2a1906c951e0ba8aed9922ab24e1f659413a06083f8a0bfea56cfff14bddef23bced449f36bcd369da72f90ddf0512e7b0801ba5a0c8eaa8eb0582c630815e992192042cfb0a7c7239f76219197c2fdf18b6553260c105280806d4f037d7b04bdf3da9fd7e9a207db5c71f7e548f4288928f047c989c4cb9cbb8088eec7bd2fa5c252e693f51a3cfc660f666af6a255a5ca0fd2216d5ccd66cbd9c11afa61067d7f615ec8d0dc0c879b5fe633d8c9443f97285da597e4da8a3993af36f0be06acfa9b8058ec70bbc78b876e4c6c5d2108fb05c15a74ba48a3d7ded697cbc1748c228d77d1e0794a41fd5240fa67c3ed745fe47555a47c3d6163d8ce95fd6c2d0d6fa48f8e5b411e571e442109b1cb200d9a8117ee08bfe645f96aca34f7b7559622bbab75143dcad59f126ae0d319e6668ebba417e725638c4febf2e",
|
||||
"siggpg": "883f0305005f2310139ec3e4c0f7e257e611023e11009f639c5fe64abaa76224dab3a9f70c2714a84c63bd009d1cc184fb4b428dfcd7c3556f4a5f860cc0187740"
|
||||
},
|
||||
{
|
||||
"name": "libgcc",
|
||||
"version": "10.0.1",
|
||||
"release": "0.11.fc32",
|
||||
"epoch": null,
|
||||
"arch": "x86_64",
|
||||
"sigmd5": "84fc907a5047aeebaf8da1642925a417",
|
||||
"sigpgp": "89023304000108001d162104963a2beb02009608fe67ea4249fd77499570ff3105025f5a272b000a091049fd77499570ff31ccdb0ffe38b95a55ebf3c021526b3cd4f2358c7e23f7767d1f5ce4b7cccef7b33653c6a96a23022313a818fbaf7abeb41837910f0d3ac15664e02838d5939d38ff459aa0076e248728a032d3ae09ddfaec955f941601081a2e3f9bbd49586fd65c1bc1b31685aeb0405687d1791471eab7359ccf00d5584ddef680e99ebc8a4846316391b9baa68ac8ed8ad696ee16fd625d847f8edd92517df3ea6920a46b77b4f119715a0f619f38835d25e0bd0eb5cfad08cd9c796eace6a2b28f4d3dee552e6068255d9748dc2a1906c951e0ba8aed9922ab24e1f659413a06083f8a0bfea56cfff14bddef23bced449f36bcd369da72f90ddf0512e7b0801ba5a0c8eaa8eb0582c630815e992192042cfb0a7c7239f76219197c2fdf18b6553260c105280806d4f037d7b04bdf3da9fd7e9a207db5c71f7e548f4288928f047c989c4cb9cbb8088eec7bd2fa5c252e693f51a3cfc660f666af6a255a5ca0fd2216d5ccd66cbd9c11afa61067d7f615ec8d0dc0c879b5fe633d8c9443f97285da597e4da8a3993af36f0be06acfa9b8058ec70bbc78b876e4c6c5d2108fb05c15a74ba48a3d7ded697cbc1748c228d77d1e0794a41fd5240fa67c3ed745fe47555a47c3d6163d8ce95fd6c2d0d6fa48f8e5b411e571e442109b1cb200d9a8117ee08bfe645f96aca34f7b7559622bbab75143dcad59f126ae0d319e6668ebba417e725638c4febf2e",
|
||||
"siggpg": null
|
||||
},
|
||||
{
|
||||
"name": "libgcc-madeup",
|
||||
"version": "10.0.1",
|
||||
"release": "0.11.fc32",
|
||||
"epoch": null,
|
||||
"arch": "x86_64",
|
||||
"sigmd5": "84fc907a5047aeebaf8da1642925a418",
|
||||
"sigpgp": null,
|
||||
"siggpg": null
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
metadata := new(osbuild.PipelineMetadata)
|
||||
err := json.Unmarshal([]byte(raw), metadata)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, metadata)
|
||||
require.Len(t, *metadata, 1)
|
||||
|
||||
rpms := OSBuildMetadataToRPMs(*metadata)
|
||||
|
||||
require.Len(t, rpms, 3)
|
||||
|
||||
signature1 := "89023304000108001d162104963a2beb02009608fe67ea4249fd77499570ff3105025f5a272b000a091049fd77499570ff31ccdb0ffe38b95a55ebf3c021526b3cd4f2358c7e23f7767d1f5ce4b7cccef7b33653c6a96a23022313a818fbaf7abeb41837910f0d3ac15664e02838d5939d38ff459aa0076e248728a032d3ae09ddfaec955f941601081a2e3f9bbd49586fd65c1bc1b31685aeb0405687d1791471eab7359ccf00d5584ddef680e99ebc8a4846316391b9baa68ac8ed8ad696ee16fd625d847f8edd92517df3ea6920a46b77b4f119715a0f619f38835d25e0bd0eb5cfad08cd9c796eace6a2b28f4d3dee552e6068255d9748dc2a1906c951e0ba8aed9922ab24e1f659413a06083f8a0bfea56cfff14bddef23bced449f36bcd369da72f90ddf0512e7b0801ba5a0c8eaa8eb0582c630815e992192042cfb0a7c7239f76219197c2fdf18b6553260c105280806d4f037d7b04bdf3da9fd7e9a207db5c71f7e548f4288928f047c989c4cb9cbb8088eec7bd2fa5c252e693f51a3cfc660f666af6a255a5ca0fd2216d5ccd66cbd9c11afa61067d7f615ec8d0dc0c879b5fe633d8c9443f97285da597e4da8a3993af36f0be06acfa9b8058ec70bbc78b876e4c6c5d2108fb05c15a74ba48a3d7ded697cbc1748c228d77d1e0794a41fd5240fa67c3ed745fe47555a47c3d6163d8ce95fd6c2d0d6fa48f8e5b411e571e442109b1cb200d9a8117ee08bfe645f96aca34f7b7559622bbab75143dcad59f126ae0d319e6668ebba417e725638c4febf2e"
|
||||
require.Equal(t, RPM{
|
||||
Type: "rpm",
|
||||
Name: "libgcc",
|
||||
Version: "10.0.1",
|
||||
Release: "0.11.fc32",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
Sigmd5: "84fc907a5047aeebaf8da1642925a417",
|
||||
Signature: &signature1,
|
||||
}, rpms[1])
|
||||
|
||||
// GPG has a priority over PGP
|
||||
signature0 := "883f0305005f2310139ec3e4c0f7e257e611023e11009f639c5fe64abaa76224dab3a9f70c2714a84c63bd009d1cc184fb4b428dfcd7c3556f4a5f860cc0187740"
|
||||
require.Equal(t, signature0, *rpms[0].Signature)
|
||||
|
||||
// if neither GPG nor PGP is set, the signature is nil
|
||||
require.Nil(t, rpms[2].Signature)
|
||||
}
|
||||
|
|
@ -1,195 +0,0 @@
|
|||
package koji
|
||||
|
||||
// BUILD METADATA
|
||||
|
||||
// TypeInfoBuild is a map whose entries are the names of the build types
|
||||
// used for the build, and the values are free-form maps containing
|
||||
// type-specific information for the build.
|
||||
type TypeInfoBuild struct {
|
||||
// Image holds extra metadata about all images built by the build.
|
||||
// It is a map whose keys are the filenames of the images, and
|
||||
// the values are the extra metadata for the image.
|
||||
// There can't be more than one image with the same filename.
|
||||
Image map[string]ImageExtraInfo `json:"image"`
|
||||
}
|
||||
|
||||
// BuildExtra holds extra metadata associated with the build.
|
||||
// It is a free-form map, but must contain at least the 'typeinfo' key.
|
||||
type BuildExtra struct {
|
||||
TypeInfo TypeInfoBuild `json:"typeinfo"`
|
||||
// Manifest holds extra metadata about osbuild manifests attached to the build.
|
||||
// It is a map whose keys are the filenames of the manifests, and
|
||||
// the values are the extra metadata for the manifest.
|
||||
Manifest map[string]*ManifestExtraInfo `json:"osbuild_manifest,omitempty"`
|
||||
}
|
||||
|
||||
// Build represents a Koji build and holds metadata about it.
|
||||
type Build struct {
|
||||
BuildID uint64 `json:"build_id"`
|
||||
TaskID uint64 `json:"task_id"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Release string `json:"release"`
|
||||
Source string `json:"source"`
|
||||
StartTime int64 `json:"start_time"`
|
||||
EndTime int64 `json:"end_time"`
|
||||
// NOTE: This is the struct that ends up shown in the buildinfo and webui in Koji.
|
||||
Extra BuildExtra `json:"extra"`
|
||||
}
|
||||
|
||||
// BUIDROOT METADATA
|
||||
|
||||
// Host holds information about the host where the build was run.
|
||||
type Host struct {
|
||||
Os string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
|
||||
// ContentGenerator holds information about the content generator which run the build.
|
||||
type ContentGenerator struct {
|
||||
Name string `json:"name"` // Must be 'osbuild'.
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// Container holds information about the container in which the build was run.
|
||||
type Container struct {
|
||||
// Type of the container that was used, e.g. 'none', 'chroot', 'kvm', 'docker', etc.
|
||||
Type string `json:"type"`
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
|
||||
// Tool holds information about a tool used to run build.
|
||||
type Tool struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// BuildRoot represents a buildroot used for the build.
|
||||
type BuildRoot struct {
|
||||
ID uint64 `json:"id"`
|
||||
Host Host `json:"host"`
|
||||
ContentGenerator ContentGenerator `json:"content_generator"`
|
||||
Container Container `json:"container"`
|
||||
Tools []Tool `json:"tools"`
|
||||
RPMs []RPM `json:"components"`
|
||||
}
|
||||
|
||||
// OUTPUT METADATA
|
||||
|
||||
type ImageOutputTypeExtraInfo interface {
|
||||
isImageOutputTypeMD()
|
||||
}
|
||||
|
||||
// OsbuildArtifact represents a configuration to produce the image using osbuild.
|
||||
type OsbuildArtifact struct {
|
||||
// Filename of the image as produced by osbuild
|
||||
ExportFilename string `json:"export_filename"`
|
||||
// Name of the osbuild pipeline, which was exported to produce this image
|
||||
ExportName string `json:"export_name"`
|
||||
}
|
||||
|
||||
// ImageExtraInfo holds extra metadata about the image.
|
||||
// This structure is shared for the Extra metadata of the output and the build.
|
||||
type ImageExtraInfo struct {
|
||||
// Koji docs say: "should contain IDs that allow tracking the output back to the system in which it was generated"
|
||||
// TODO: we should probably add some ID here, probably the OSBuildJob UUID?
|
||||
|
||||
Arch string `json:"arch"`
|
||||
// Boot mode of the image
|
||||
BootMode string `json:"boot_mode,omitempty"`
|
||||
// Configuration used to produce this image using osbuild
|
||||
OSBuildArtifact *OsbuildArtifact `json:"osbuild_artifact,omitempty"`
|
||||
// Version of the osbuild binary used by the worker to build the image
|
||||
OSBuildVersion string `json:"osbuild_version,omitempty"`
|
||||
// Results from any upload targets associated with the image
|
||||
// The structure of the data does not need to be known by the consumer,
|
||||
// it is just a list of JSON objects.
|
||||
UploadTargetResults []interface{} `json:"upload_target_results,omitempty"`
|
||||
}
|
||||
|
||||
func (ImageExtraInfo) isImageOutputTypeMD() {}
|
||||
|
||||
type OSBuildComposerDepModule struct {
|
||||
Path string `json:"path"`
|
||||
Version string `json:"version"`
|
||||
Replace *OSBuildComposerDepModule `json:"replace,omitempty"`
|
||||
}
|
||||
|
||||
// ManifestInfo holds information about the environment in which
|
||||
// the manifest was produced and which could affect its content.
|
||||
type ManifestInfo struct {
|
||||
OSBuildComposerVersion string `json:"osbuild_composer_version"`
|
||||
// List of relevant modules used by osbuild-composer which
|
||||
// could affect the manifest content.
|
||||
OSBuildComposerDeps []*OSBuildComposerDepModule `json:"osbuild_composer_deps,omitempty"`
|
||||
}
|
||||
|
||||
// ManifestExtraInfo holds extra metadata about the osbuild manifest.
|
||||
type ManifestExtraInfo struct {
|
||||
Arch string `json:"arch"`
|
||||
Info *ManifestInfo `json:"info,omitempty"`
|
||||
}
|
||||
|
||||
func (ManifestExtraInfo) isImageOutputTypeMD() {}
|
||||
|
||||
type SbomDocExtraInfo struct {
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
|
||||
func (SbomDocExtraInfo) isImageOutputTypeMD() {}
|
||||
|
||||
// BuildOutputExtra holds extra metadata associated with the build output.
|
||||
type BuildOutputExtra struct {
|
||||
// ImageOutput holds extra metadata about a single "image" output.
|
||||
// "image" in this context is the "build type" in the Koji terminology,
|
||||
// not necessarily an actual image. It can and must be used also for
|
||||
// other supplementary files related to the image, such as osbuild manifest.
|
||||
// The only exception are logs, which do not need to specify any "typeinfo".
|
||||
ImageOutput ImageOutputTypeExtraInfo `json:"image"`
|
||||
}
|
||||
|
||||
// BuildOutputType represents the type of a BuildOutput.
|
||||
type BuildOutputType string
|
||||
|
||||
const (
|
||||
BuildOutputTypeImage BuildOutputType = "image"
|
||||
BuildOutputTypeLog BuildOutputType = "log"
|
||||
BuildOutputTypeManifest BuildOutputType = "osbuild-manifest"
|
||||
BuildOutputTypeSbomDoc BuildOutputType = "sbom-doc"
|
||||
)
|
||||
|
||||
// ChecksumType represents the type of a checksum used for a BuildOutput.
|
||||
type ChecksumType string
|
||||
|
||||
const (
|
||||
ChecksumTypeMD5 ChecksumType = "md5"
|
||||
ChecksumTypeAdler32 ChecksumType = "adler32"
|
||||
ChecksumTypeSHA256 ChecksumType = "sha256"
|
||||
)
|
||||
|
||||
// BuildOutput represents an output from the OSBuild content generator.
|
||||
// The output can be a file of various types, which is imported to Koji.
|
||||
// Examples of types are "image", "log" or other.
|
||||
type BuildOutput struct {
|
||||
BuildRootID uint64 `json:"buildroot_id"`
|
||||
Filename string `json:"filename"`
|
||||
FileSize uint64 `json:"filesize"`
|
||||
Arch string `json:"arch"` // can be 'noarch' or a specific arch
|
||||
ChecksumType ChecksumType `json:"checksum_type"`
|
||||
Checksum string `json:"checksum"`
|
||||
Type BuildOutputType `json:"type"`
|
||||
RPMs []RPM `json:"components,omitempty"`
|
||||
Extra *BuildOutputExtra `json:"extra,omitempty"`
|
||||
}
|
||||
|
||||
// CONTENT GENERATOR METADATA
|
||||
|
||||
// Metadata holds Koji Content Generator metadata.
|
||||
// This is passed to the CGImport call.
|
||||
// For more information, see https://docs.pagure.org/koji/content_generator_metadata/
|
||||
type Metadata struct {
|
||||
MetadataVersion int `json:"metadata_version"` // must be '0'
|
||||
Build Build `json:"build"`
|
||||
BuildRoots []BuildRoot `json:"buildroots"`
|
||||
Outputs []BuildOutput `json:"output"`
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue